Package mvpa :: Package clfs :: Module transerror

[hide private]
[frames] | no frames]

# Source Code for Module mvpa.clfs.transerror

       1  # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
       2  # vi: set ft=python sts=4 ts=4 sw=4 et:
       3  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
       4  #
       5  #   See COPYING file distributed along with the PyMVPA package for the
       6  #   copyright and license terms.
       7  #
       8  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
       9  """Utility class to compute the transfer error of classifiers."""
      10
      11  __docformat__ = 'restructuredtext'
      12
      13  import mvpa.support.copy as copy
      14
      15  import numpy as N
      16
      17  from StringIO import StringIO
      18  from math import log10, ceil
      19
      20  from mvpa.base import externals
      21
      22  from mvpa.misc.errorfx import meanPowerFx, rootMeanPowerFx, RMSErrorFx, \
      23       CorrErrorFx, CorrErrorPFx, RelativeRMSErrorFx, MeanMismatchErrorFx, \
      24       AUCErrorFx
      25  from mvpa.base import warning
      26  from mvpa.misc.state import StateVariable, ClassWithCollections
      27  from mvpa.base.dochelpers import enhancedDocString, table2string
      28  from mvpa.clfs.stats import autoNullDist
      29
      30  if __debug__:
      31      from mvpa.base import debug
      32
      33  if externals.exists('scipy'):
      34      from scipy.stats.stats import nanmean
      35  else:
      36      from mvpa.clfs.stats import nanmean
      37
      38 -def _p2(x, prec=2):
      39      """Helper to print depending on the type nicely. For some
      40      reason %.2g for 100 prints exponential form which is ugly
      41      """
      42      if isinstance(x, int):
      43          return "%d" % x
      44      elif isinstance(x, float):
      45          s = ("%%.%df" % prec % x).rstrip('0').rstrip('.').lstrip()
      46          if s == '':
      47              s = '0'
      48          return s
      49      else:
      50          return "%s" % x
      51
      52
      53
      54 -class SummaryStatistics(object):
      55      """Basic class to collect targets/predictions and report summary statistics
      56
      57      It takes care about collecting the sets, which are just tuples
      58      (targets, predictions, values). While 'computing' the matrix, all
      59      sets are considered together.  Children of the class are
      60      responsible for computation and display.
      61      """
      62
      63      _STATS_DESCRIPTION = (
      64          ('# of sets',
      65           'number of target/prediction sets which were provided',
      66           None), )
      67
      68
      69 -    def __init__(self, targets=None, predictions=None, values=None, sets=None):
      70          """Initialize SummaryStatistics
      71
      72          targets or predictions cannot be provided alone (ie targets
      73          without predictions)
      74
      75          :Parameters:
      76           targets
      77             Optional set of targets
      78           predictions
      79             Optional set of predictions
      80           values
      81             Optional set of values (which served for prediction)
      82           sets
      83             Optional list of sets
      84          """
      85          self._computed = False
      86          """Flag either it was computed for a given set of data"""
      87
      88          self.__sets = (sets, [])[int(sets is None)]
      89          """Datasets (target, prediction) to compute confusion matrix on"""
      90
      91          self._stats = {}
      92          """Dictionary to keep statistics. Initialized here to please pylint"""
      93
      94          if not targets is None or not predictions is None:
      95              if not targets is None and not predictions is None:
      96                  self.add(targets=targets, predictions=predictions,
      97                           values=values)
      98              else:
      99                  raise ValueError, \
     100                        "Please provide none or both targets and predictions"
     101
     102
     103 -    def add(self, targets, predictions, values=None):
     104          """Add new results to the set of known results"""
     105          if len(targets) != len(predictions):
     106              raise ValueError, \
     107                    "Targets[%d] and predictions[%d]" % (len(targets),
     108                                                         len(predictions)) + \
     109                    " have different number of samples"
     110
     111          if values is not None and len(targets) != len(values):
     112              raise ValueError, \
     113                    "Targets[%d] and values[%d]" % (len(targets),
     114                                                    len(values)) + \
     115                    " have different number of samples"
     116
     117          # enforce labels in predictions to be of the same datatype as in
     118          # targets, since otherwise we are getting doubles for unknown at a
     119          # given moment labels
     120          nonetype = type(None)
     121          for i in xrange(len(targets)):
     122              t1, t2 = type(targets[i]), type(predictions[i])
     123              # if there were no prediction made - leave None, otherwise
     124              # convert to appropriate type
     125              if t1 != t2 and t2 != nonetype:
     126                  #warning("Obtained target %s and prediction %s are of " %
     127                  #       (t1, t2) + "different datatypes.")
     128                  if isinstance(predictions, tuple):
     129                      predictions = list(predictions)
     130                  predictions[i] = t1(predictions[i])
     131
     132          if values is not None:
     133              # assure that we have a copy, or otherwise further in-place
     134              # modifications might screw things up (some classifiers share
     135              # values and spit out results)
     136              values = copy.deepcopy(values)
     137
     138          self.__sets.append( (targets, predictions, values) )
     139          self._computed = False
     140
     141
     142 -    def asstring(self, short=False, header=True, summary=True,
     143                   description=False):
     144          """'Pretty print' the matrix
     145
     146          :Parameters:
     147            short : bool
     148              if True, ignores the rest of the parameters and provides consise
     149              1 line summary
     150            header : bool
     151              print header of the table
     152            summary : bool
     153              print summary (accuracy)
     154            description : bool
     155              print verbose description of presented statistics
     156          """
     157          raise NotImplementedError
     158
     159
     160 -    def __str__(self):
     161          """String summary over the `SummaryStatistics`
     162
     163          It would print description of the summary statistics if 'CM'
     164          debug target is active
     165          """
     166          if __debug__:
     167              description = ('CM' in debug.active)
     168          else:
     169              description = False
     170          return self.asstring(short=False, header=True, summary=True,
     171                               description=description)
     172
     173
     174 -    def __iadd__(self, other):
     175          """Add the sets from `other` s `SummaryStatistics` to current one
     176          """
     177          #print "adding ", other, " to ", self
     178          # need to do shallow copy, or otherwise smth like "cm += cm"
     179          # would loop forever and exhaust memory eventually
     180          othersets = copy.copy(other.__sets)
     181          for set in othersets:
     182              self.add(*set)#[0], set[1])
     183          return self
     184
     185
     186 -    def __add__(self, other):
     187          """Add two `SummaryStatistics`s
     188          """
     189          result = copy.copy(self)
     190          result += other
     191          return result
     192
     193
     194 -    def compute(self):
     195          """Actually compute the confusion matrix based on all the sets"""
     196          if self._computed:
     197              return
     198
     199          self._compute()
     200          self._computed = True
     201
     202
     203 -    def _compute(self):
     204          """Compute basic statistics
     205          """
     206          self._stats = {'# of sets' : len(self.sets)}
     207
     208
     209      @property
     210 -    def summaries(self):
     211          """Return a list of separate summaries per each stored set"""
     212          return [ self.__class__(sets=[x]) for x in self.sets ]
     213
     214
     215      @property
     216 -    def error(self):
     217          raise NotImplementedError
     218
     219
     220      @property
     221 -    def stats(self):
     222          self.compute()
     223          return self._stats
     224
     225
     226 -    def reset(self):
     227          """Cleans summary -- all data/sets are wiped out
     228          """
     229          self.__sets = []
     230          self._computed = False
     231
     232
     233      sets = property(lambda self:self.__sets)
     234
     235
     236 -class ROCCurve(object):
     237      """Generic class for ROC curve computation and plotting
     238      """
     239
     240 -    def __init__(self, labels, sets=None):
     241          """
     242          :Parameters:
     243            labels : list
     244              labels which were used (in order of values if multiclass,
     245              or 1 per class for binary problems (e.g. in SMLR))
     246            sets : list of tuples
     247              list of sets for the analysis
     248          """
     249          self._labels = labels
     250          self._sets = sets
     251          self.__computed = False
     252
     253
     254 -    def _compute(self):
     255          """Lazy computation if needed
     256          """
     257          if self.__computed:
     258              return
     259          # local bindings
     260          labels = self._labels
     261          Nlabels = len(labels)
     262          sets = self._sets
     263
     264          # Handle degenerate cases politely
     265          if Nlabels < 2:
     266              warning("ROC was asked to be evaluated on data with %i"
     267                      " labels which is a degenerate case.")
     268              self._ROCs = []
     269              self._aucs = []
     270              return
     271
     272          # take sets which have values in the shape we can handle
     273          def _checkValues(set_):
     274              """Check if values are 'acceptable'"""
     275              if len(set_)<3: return False
     276              x = set_[2]
     277              # TODO: OPT: need optimization
     278              if (x is None) or len(x) == 0: return False          # undefined
     279              for v in x:
     280                  try:
     281                      if Nlabels <= 2 and N.isscalar(v):
     282                          continue
     283                      if (isinstance(v, dict) or # not dict for pairs
     284                          ((Nlabels>=2) and len(v)!=Nlabels) # 1 per each label for multiclass
     285                          ): return False
     286                  except Exception, e:
     287                      # Something else which is not supported, like
     288                      # in shogun interface we don't yet extract values per each label or
     289                      # in pairs in the case of built-in multiclass
     290                      if __debug__:
     291                          debug('ROC', "Exception %s while checking "
     292                                "either %s are valid labels" % (str(e), x))
     293                      return False
     294              return True
     295
     296          sets_wv = filter(_checkValues, sets)
     297          # check if all had values, if not -- complain
     298          Nsets_wv = len(sets_wv)
     299          if Nsets_wv > 0 and len(sets) != Nsets_wv:
     300              warning("Only %d sets have values assigned from %d sets. "
     301                      "ROC estimates might be incorrect." %
     302                      (Nsets_wv, len(sets)))
     303          # bring all values to the same 'shape':
     304          #  1 value per each label. In binary classifier, if only a single
     305          #  value is provided, add '0' for 0th label 'value'... it should
     306          #  work taking drunk Yarik logic ;-)
     307          # yoh: apparently it caused problems whenever we had just a single
     308          #      unique label in the sets. Introduced handling for
     309          #      NLabels == 1
     310          for iset,s in enumerate(sets_wv):
     311              # we will do inplace modification, thus go by index
     312              values = s[2]
     313              # we would need it to be a list to reassign element with a list
     314              if isinstance(values, N.ndarray) and len(values.shape)==1:
     315                  # XXX ??? so we are going away from inplace modifications?
     316                  values = list(values)
     317              rangev = None
     318              for i in xrange(len(values)):
     319                  v = values[i]
     320                  if N.isscalar(v):
     321                      if Nlabels == 1:
     322                          # ensure the right dimensionality
     323                          values[i] = N.array(v, ndmin=2)
     324                      elif Nlabels == 2:
     325                          def last_el(x):
     326                              """Helper function. Returns x if x is scalar, and
     327                              last element if x is not (ie list/tuple)"""
     328                              if N.isscalar(x): return x
     329                              else:             return x[-1]
     330                          if rangev is None:
     331                              # we need to figure out min/max values
     332                              # to invert for the 0th label
     333                              values_ = [last_el(x) for x in values]
     334                              rangev = N.min(values_) + N.max(values_)
     335                          values[i] = [rangev - v, v]
     336                      else:
     337                          raise ValueError, \
     338                                "Cannot have a single 'value' for multiclass" \
     339                                " classification. Got %s" % (v)
     340                  elif len(v) != Nlabels:
     341                      raise ValueError, \
     342                            "Got %d values whenever there is %d labels" % \
     343                            (len(v), Nlabels)
     344              # reassign possibly adjusted values
     345              sets_wv[iset] = (s[0], s[1], N.asarray(values))
     346
     347
     348          # we need to estimate ROC per each label
     349          # XXX order of labels might not correspond to the one among 'values'
     350          #     which were used to make a decision... check
     351          ROCs, aucs = [], []             # 1 per label
     352          for i,label in enumerate(labels):
     353              aucs_pl = []
     354              ROCs_pl = []
     355              for s in sets_wv:
     356                  targets_pl = (N.asanyarray(s[0]) == label).astype(int)
     357                  # XXX we might unify naming between AUC/ROC
     358                  ROC = AUCErrorFx()
     359                  aucs_pl += [ROC([N.asanyarray(x)[i] for x in s[2]], targets_pl)]
     360                  ROCs_pl.append(ROC)
     361              if len(aucs_pl)>0:
     362                  ROCs += [ROCs_pl]
     363                  aucs += [nanmean(aucs_pl)]
     364                  #aucs += [N.mean(aucs_pl)]
     365
     366          # store results within the object
     367          self._ROCs =  ROCs
     368          self._aucs = aucs
     369          self.__computed = True
     370
     371
     372      @property
     373 -    def aucs(self):
     374          """Compute and return set of AUC values 1 per label
     375          """
     376          self._compute()
     377          return self._aucs
     378
     379
     380      @property
     381 -    def ROCs(self):
     382          self._compute()
     383          return self._ROCs
     384
     385
     386 -    def plot(self, label_index=0):
     387          """
     388
     389          TODO: make it friendly to labels given by values?
     390                should we also treat labels_map?
     391          """
     392          externals.exists("pylab", raiseException=True)
     393          import pylab as P
     394
     395          self._compute()
     396
     397          labels = self._labels
     398          # select only ROCs for the given label
     399          ROCs = self.ROCs[label_index]
     400
     401          fig = P.gcf()
     402          ax = P.gca()
     403
     404          P.plot([0, 1], [0, 1], 'k:')
     405
     406          for ROC in ROCs:
     407              P.plot(ROC.fp, ROC.tp, linewidth=1)
     408
     409          P.axis((0.0, 1.0, 0.0, 1.0))
     410          P.axis('scaled')
     411          P.title('Label %s. Mean AUC=%.2f' % (label_index, self.aucs[label_index]))
     412
     413          P.xlabel('False positive rate')
     414          P.ylabel('True positive rate')
     415
     416
     417 -class ConfusionMatrix(SummaryStatistics):
     418      """Class to contain information and display confusion matrix.
     419
     420      Implementation of the `SummaryStatistics` in the case of
     421      classification problem. Actual computation of confusion matrix is
     422      delayed until all data is acquired (to figure out complete set of
     423      labels). If testing data doesn't have a complete set of labels,
     424      but you like to include all labels, provide them as a parameter to
     425      the constructor.
     426
     427      Confusion matrix provides a set of performance statistics (use
     428      asstring(description=True) for the description of abbreviations),
     429      as well ROC curve (http://en.wikipedia.org/wiki/ROC_curve)
     430      plotting and analysis (AUC) in the limited set of problems:
     431      binary, multiclass 1-vs-all.
     432      """
     433
     434      _STATS_DESCRIPTION = (
     435          ('TP', 'true positive (AKA hit)', None),
     436          ('TN', 'true negative (AKA correct rejection)', None),
     437          ('FP', 'false positive (AKA false alarm, Type I error)', None),
     438          ('FN', 'false negative (AKA miss, Type II error)', None),
     439          ('TPR', 'true positive rate (AKA hit rate, recall, sensitivity)',
     440                  'TPR = TP / P = TP / (TP + FN)'),
     441          ('FPR', 'false positive rate (AKA false alarm rate, fall-out)',
     442                  'FPR = FP / N = FP / (FP + TN)'),
     443          ('ACC', 'accuracy', 'ACC = (TP + TN) / (P + N)'),
     444          ('SPC', 'specificity', 'SPC = TN / (FP + TN) = 1 - FPR'),
     445          ('PPV', 'positive predictive value (AKA precision)',
     446                  'PPV = TP / (TP + FP)'),
     447          ('NPV', 'negative predictive value', 'NPV = TN / (TN + FN)'),
     448          ('FDR', 'false discovery rate', 'FDR = FP / (FP + TP)'),
     449          ('MCC', "Matthews Correlation Coefficient",
     450                  "MCC = (TP*TN - FP*FN)/sqrt(P N P' N')"),
     451          ('AUC', "Area under (AUC) curve", None),
     452          ) + SummaryStatistics._STATS_DESCRIPTION
     453
     454
     455 -    def __init__(self, labels=None, labels_map=None, **kwargs):
     456          """Initialize ConfusionMatrix with optional list of `labels`
     457
     458          :Parameters:
     459           labels : list
     460             Optional set of labels to include in the matrix
     461           labels_map : None or dict
     462             Dictionary from original dataset to show mapping into
     463             numerical labels
     464           targets
     465             Optional set of targets
     466           predictions
     467             Optional set of predictions
     468             """
     469
     470          SummaryStatistics.__init__(self, **kwargs)
     471
     472          if labels == None:
     473              labels = []
     474          self.__labels = labels
     475          """List of known labels"""
     476          self.__labels_map = labels_map
     477          """Mapping from original into given labels"""
     478          self.__matrix = None
     479          """Resultant confusion matrix"""
     480
     481
     482      # XXX might want to remove since summaries does the same, just without
     483      #     supplying labels
     484      @property
     485 -    def matrices(self):
     486          """Return a list of separate confusion matrix per each stored set"""
     487          return [ self.__class__(labels=self.labels,
     488                                  labels_map=self.labels_map,
     489                                  sets=[x]) for x in self.sets]
     490
     491
     492 -    def _compute(self):
     493          """Actually compute the confusion matrix based on all the sets"""
     494
     495          super(ConfusionMatrix, self)._compute()
     496
     497          if __debug__:
     498              if not self.__matrix is None:
     499                  debug("LAZY",
     500                        "Have to recompute %s#%s" \
     501                          % (self.__class__.__name__, id(self)))
     502
     503
     504          # TODO: BinaryClassifier might spit out a list of predictions for each
     505          # value need to handle it... for now just keep original labels
     506          try:
     507              # figure out what labels we have
     508              labels = \
     509                  list(reduce(lambda x, y: x.union(set(y[0]).union(set(y[1]))),
     510                              self.sets,
     511                              set(self.__labels)))
     512          except:
     513              labels = self.__labels
     514
     515          # Check labels_map if it was provided if it covers all the labels
     516          labels_map = self.__labels_map
     517          if labels_map is not None:
     518              labels_set = set(labels)
     519              map_labels_set = set(labels_map.values())
     520
     521              if not map_labels_set.issuperset(labels_set):
     522                  warning("Provided labels_map %s is not coherent with labels "
     523                          "provided to ConfusionMatrix. No reverse mapping "
     524                          "will be provided" % labels_map)
     525                  labels_map = None
     526
     527          # Create reverse map
     528          labels_map_rev = None
     529          if labels_map is not None:
     530              labels_map_rev = {}
     531              for k,v in labels_map.iteritems():
     532                  v_mapping = labels_map_rev.get(v, [])
     533                  v_mapping.append(k)
     534                  labels_map_rev[v] = v_mapping
     535          self.__labels_map_rev = labels_map_rev
     536
     537          labels.sort()
     538          self.__labels = labels          # store the recomputed labels
     539
     540          Nlabels, Nsets = len(labels), len(self.sets)
     541
     542          if __debug__:
     543              debug("CM", "Got labels %s" % labels)
     544
     545          # Create a matrix for all votes
     546          mat_all = N.zeros( (Nsets, Nlabels, Nlabels), dtype=int )
     547
     548          # create total number of samples of each label counts
     549          # just for convinience I guess since it can always be
     550          # computed from mat_all
     551          counts_all = N.zeros( (Nsets, Nlabels) )
     552
     553          # reverse mapping from label into index in the list of labels
     554          rev_map = dict([ (x[1], x[0]) for x in enumerate(labels)])
     555          for iset, set_ in enumerate(self.sets):
     556              for t,p in zip(*set_[:2]):
     557                  mat_all[iset, rev_map[p], rev_map[t]] += 1
     558
     559
     560          # for now simply compute a sum of votes across different sets
     561          # we might do something more sophisticated later on, and this setup
     562          # should easily allow it
     563          self.__matrix = N.sum(mat_all, axis=0)
     564          self.__Nsamples = N.sum(self.__matrix, axis=0)
     565          self.__Ncorrect = sum(N.diag(self.__matrix))
     566
     567          TP = N.diag(self.__matrix)
     568          offdiag = self.__matrix - N.diag(TP)
     569          stats = {
     570              '# of labels' : Nlabels,
     571              'TP' : TP,
     572              'FP' : N.sum(offdiag, axis=1),
     573              'FN' : N.sum(offdiag, axis=0)}
     574
     575          stats['CORR']  = N.sum(TP)
     576          stats['TN']  = stats['CORR'] - stats['TP']
     577          stats['P']  = stats['TP'] + stats['FN']
     578          stats['N']  = N.sum(stats['P']) - stats['P']
     579          stats["P'"] = stats['TP'] + stats['FP']
     580          stats["N'"] = stats['TN'] + stats['FN']
     581          stats['TPR'] = stats['TP'] / (1.0*stats['P'])
     582          # reset nans in TPRs to 0s whenever there is no entries
     583          # for those labels among the targets
     584          stats['TPR'][stats['P'] == 0] = 0
     585          stats['PPV'] = stats['TP'] / (1.0*stats["P'"])
     586          stats['NPV'] = stats['TN'] / (1.0*stats["N'"])
     587          stats['FDR'] = stats['FP'] / (1.0*stats["P'"])
     588          stats['SPC'] = (stats['TN']) / (1.0*stats['FP'] + stats['TN'])
     589
     590          MCC_denom = N.sqrt(1.0*stats['P']*stats['N']*stats["P'"]*stats["N'"])
     591          nz = MCC_denom!=0.0
     592          stats['MCC'] = N.zeros(stats['TP'].shape)
     593          stats['MCC'][nz] = \
     594                   (stats['TP'] * stats['TN'] - stats['FP'] * stats['FN'])[nz] \
     595                    / MCC_denom[nz]
     596
     597          stats['ACC'] = N.sum(TP)/(1.0*N.sum(stats['P']))
     598          stats['ACC%'] = stats['ACC'] * 100.0
     599
     600          #
     601          # ROC computation if available
     602          ROC = ROCCurve(labels=labels, sets=self.sets)
     603          aucs = ROC.aucs
     604          if len(aucs)>0:
     605              stats['AUC'] = aucs
     606              if len(aucs) != Nlabels:
     607                  raise RuntimeError, \
     608                        "We must got a AUC per label. Got %d instead of %d" % \
     609                        (len(aucs), Nlabels)
     610              self.ROC = ROC
     611          else:
     612              # we don't want to provide ROC if it is bogus
     613              stats['AUC'] = [N.nan] * Nlabels
     614              self.ROC = None
     615
     616
     617          # compute mean stats
     618          for k,v in stats.items():
     619              stats['mean(%s)' % k] = N.mean(v)
     620
     621          self._stats.update(stats)
     622
     623
     624 -    def asstring(self, short=False, header=True, summary=True,
     625                   description=False):
     626          """'Pretty print' the matrix
     627
     628          :Parameters:
     629            short : bool
     630              if True, ignores the rest of the parameters and provides consise
     631              1 line summary
     632            header : bool
     633              print header of the table
     634            summary : bool
     635              print summary (accuracy)
     636            description : bool
     637              print verbose description of presented statistics
     638          """
     639          if len(self.sets) == 0:
     640              return "Empty"
     641
     642          self.compute()
     643
     644          # some shortcuts
     645          labels = self.__labels
     646          labels_map_rev = self.__labels_map_rev
     647          matrix = self.__matrix
     648
     649          labels_rev = []
     650          if labels_map_rev is not None:
     651              labels_rev = [','.join([str(x) for x in labels_map_rev[l]])
     652                                     for l in labels]
     653
     654          out = StringIO()
     655          # numbers of different entries
     656          Nlabels = len(labels)
     657          Nsamples = self.__Nsamples.astype(int)
     658
     659          stats = self._stats
     660          if short:
     661              return "%(# of sets)d sets %(# of labels)d labels " \
     662                     " ACC:%(ACC).2f" \
     663                     % stats
     664
     665          Ndigitsmax = int(ceil(log10(max(Nsamples))))
     666          Nlabelsmax = max( [len(str(x)) for x in labels] )
     667
     668          # length of a single label/value
     669          L = max(Ndigitsmax+2, Nlabelsmax) #, len("100.00%"))
     670          res = ""
     671
     672          stats_perpredict = ["P'", "N'", 'FP', 'FN', 'PPV', 'NPV', 'TPR',
     673                              'SPC', 'FDR', 'MCC']
     674          # print AUC only if ROC was computed
     675          if self.ROC is not None: stats_perpredict += [ 'AUC' ]
     676          stats_pertarget = ['P', 'N', 'TP', 'TN']
     677          stats_summary = ['ACC', 'ACC%', '# of sets']
     678
     679
     680          #prefixlen = Nlabelsmax + 2 + Ndigitsmax + 1
     681          prefixlen = Nlabelsmax + 1
     682          pref = ' '*(prefixlen) # empty prefix
     683
     684          if matrix.shape != (Nlabels, Nlabels):
     685              raise ValueError, \
     686                    "Number of labels %d doesn't correspond the size" + \
     687                    " of a confusion matrix %s" % (Nlabels, matrix.shape)
     688
     689          # list of lists of what is printed
     690          printed = []
     691          underscores = [" %s" % ("-" * L)] * Nlabels
     692          if header:
     693              # labels
     694              printed.append(['@l----------.        '] + labels_rev)
     695              printed.append(['@lpredictions\\targets'] + labels)
     696              # underscores
     697              printed.append(['@l            `------'] \
     698                             + underscores + stats_perpredict)
     699
     700          # matrix itself
     701          for i, line in enumerate(matrix):
     702              l = labels[i]
     703              if labels_rev != []:
     704                  l = '@r%10s / %s' % (labels_rev[i], l)
     705              printed.append(
     706                  [l] +
     707                  [ str(x) for x in line ] +
     708                  [ _p2(stats[x][i]) for x in stats_perpredict])
     709
     710          if summary:
     711              ## Various alternative schemes ;-)
     712              # printed.append([''] + underscores)
     713              # printed.append(['@lPer target \ Means:'] + underscores + \
     714              #               [_p2(x) for x in mean_stats])
     715              # printed.append(['Means:'] + [''] * len(labels)
     716              #                + [_p2(x) for x in mean_stats])
     717              printed.append(['@lPer target:'] + underscores)
     718              for stat in stats_pertarget:
     719                  printed.append([stat] + [
     720                      _p2(stats[stat][i]) for i in xrange(Nlabels)])
     721
     722              # compute mean stats
     723              # XXX refactor to expose them in stats as well, as
     724              #     mean(FCC)
     725              mean_stats = N.mean(N.array([stats[k] for k in stats_perpredict]),
     726                                  axis=1)
     727              printed.append(['@lSummary \ Means:'] + underscores
     728                             + [_p2(stats['mean(%s)' % x])
     729                                for x in stats_perpredict])
     730
     731              for stat in stats_summary:
     732                  printed.append([stat] + [_p2(stats[stat])])
     733
     734          table2string(printed, out)
     735
     736          if description:
     737              out.write("\nStatistics computed in 1-vs-rest fashion per each " \
     738                        "target.\n")
     739              out.write("Abbreviations (for details see " \
     740                        "http://en.wikipedia.org/wiki/ROC_curve):\n")
     741              for d, val, eq in self._STATS_DESCRIPTION:
     742                  out.write(" %-3s: %s\n" % (d, val))
     743                  if eq is not None:
     744                      out.write("      " + eq + "\n")
     745
     746          #out.write("%s" % printed)
     747          result = out.getvalue()
     748          out.close()
     749          return result
     750
     751
     752 -    def plot(self, labels=None, numbers=False, origin='upper',
     753               numbers_alpha=None, xlabels_vertical=True, numbers_kwargs={},
     754               **kwargs):
     755          """Provide presentation of confusion matrix in image
     756
     757          :Parameters:
     758            labels : list of int or basestring
     759              Optionally provided labels guarantee the order of
     760              presentation. Also value of None places empty column/row,
     761              thus provides visual groupping of labels (Thanks Ingo)
     762            numbers : bool
     763              Place values inside of confusion matrix elements
     764            numbers_alpha : None or float
     765              Controls textual output of numbers. If None -- all numbers
     766              are plotted in the same intensity. If some float -- it controls
     767              alpha level -- higher value would give higher contrast. (good
     768              value is 2)
     769            origin : basestring
     770              Which left corner diagonal should start
     771            xlabels_vertical : bool
     772              Either to plot xlabels vertical (benefitial if number of labels
     773              is large)
     774            numbers_kwargs : dict
     775              Additional keyword parameters to be added to numbers (if numbers
     776              is True)
     777            **kwargs
     778              Additional arguments given to imshow (\eg me cmap)
     779
     780          :Returns:
     781             (fig, im, cb) -- figure, imshow, colorbar
     782          """
     783
     784          externals.exists("pylab", raiseException=True)
     785          import pylab as P
     786
     787          self.compute()
     788          labels_order = labels
     789
     790          # some shortcuts
     791          labels = self.__labels
     792          labels_map = self.__labels_map
     793          labels_map_rev = self.__labels_map_rev
     794          matrix = self.__matrix
     795
     796          # craft original mapping from a label into index in the matrix
     797          labels_indexes = dict([(x,i) for i,x in enumerate(labels)])
     798
     799          labels_rev = []
     800          if labels_map_rev is not None:
     801              labels_rev = [','.join([str(x) for x in labels_map_rev[l]])
     802                                     for l in labels]
     803              labels_map_full = dict(zip(labels_rev, labels))
     804
     805          if labels_order is not None:
     806              labels_order_filtered = filter(lambda x:x is not None, labels_order)
     807              labels_order_filtered_set = set(labels_order_filtered)
     808              # Verify if all labels provided in labels
     809              if set(labels) == labels_order_filtered_set:
     810                  # We were provided numerical (most probably) set
     811                  labels_plot = labels_order
     812              elif len(labels_rev) \
     813                       and set(labels_rev) == labels_order_filtered_set:
     814                  # not clear if right whenever there were multiple labels
     815                  # mapped into the same
     816                  labels_plot = []
     817                  for l in labels_order:
     818                      v = None
     819                      if l is not None: v = labels_map_full[l]
     820                      labels_plot += [v]
     821              else:
     822                  raise ValueError, \
     823                        "Provided labels %s do not match set of known " \
     824                        "original labels (%s) or mapped labels (%s)" % \
     825                        (labels_order, labels, labels_rev)
     826          else:
     827              labels_plot = labels
     828
     829          # where we have Nones?
     830          isempty = N.array([l is None for l in labels_plot])
     831          non_empty = N.where(N.logical_not(isempty))[0]
     832          # numbers of different entries
     833          NlabelsNN = len(non_empty)
     834          Nlabels = len(labels_plot)
     835
     836          if matrix.shape != (NlabelsNN, NlabelsNN):
     837              raise ValueError, \
     838                    "Number of labels %d doesn't correspond the size" + \
     839                    " of a confusion matrix %s" % (NlabelsNN, matrix.shape)
     840
     841          confusionmatrix = N.zeros((Nlabels, Nlabels))
     842          mask = confusionmatrix.copy()
     843          ticks = []
     844          tick_labels = []
     845          # populate in a silly way
     846          reordered_indexes = [labels_indexes[i] for i in labels_plot
     847                               if i is not None]
     848          for i, l in enumerate(labels_plot):
     849              if l is not None:
     850                  j = labels_indexes[l]
     851                  confusionmatrix[i, non_empty] = matrix[j, reordered_indexes]
     852                  confusionmatrix[non_empty, i] = matrix[reordered_indexes, j]
     853                  ticks += [i + 0.5]
     854                  if labels_map_rev is not None:
     855                      tick_labels += ['/'.join(labels_map_rev[l])]
     856                  else:
     857                      tick_labels += [str(l)]
     858              else:
     859                  mask[i, :] = mask[:, i] = 1
     860
     861          confusionmatrix = N.ma.MaskedArray(confusionmatrix, mask=mask)
     862
     863          # turn off automatic update if interactive
     864          if P.matplotlib.get_backend() == 'TkAgg':
     865              P.ioff()
     866
     867          fig = P.gcf()
     868          ax = P.gca()
     869          ax.axis('off')
     870
     871          # some customization depending on the origin
     872          xticks_position, yticks, ybottom = {
     873              'upper': ('top', [Nlabels-x for x in ticks], 0.1),
     874              'lower': ('bottom', ticks, 0.2)
     875              }[origin]
     876
     877
     878          # Plot
     879          axi = fig.add_axes([0.15, ybottom, 0.7, 0.7])
     880          im = axi.imshow(confusionmatrix, interpolation="nearest", origin=origin,
     881                          aspect='equal', extent=(0, Nlabels, 0, Nlabels),
     882                          **kwargs)
     883
     884          # plot numbers
     885          if numbers:
     886              numbers_kwargs_ = {'fontsize': 10,
     887                                 'horizontalalignment': 'center',
     888                                 'verticalalignment': 'center'}
     889              maxv = float(N.max(confusionmatrix))
     890              colors = [im.to_rgba(0), im.to_rgba(maxv)]
     891              for i,j in zip(*N.logical_not(mask).nonzero()):
     892                  v = confusionmatrix[j, i]
     893                  # scale alpha non-linearly
     894                  if numbers_alpha is None:
     895                      alpha = 1.0
     896                  else:
     897                      # scale according to value
     898                      alpha = 1 - N.array(1 - v / maxv) ** numbers_alpha
     899                  y = {'lower':j, 'upper':Nlabels-j-1}[origin]
     900                  numbers_kwargs_['color'] = colors[int(v<maxv/2)]
     901                  numbers_kwargs_.update(numbers_kwargs)
     902                  P.text(i+0.5, y+0.5, '%d' % v, alpha=alpha, **numbers_kwargs_)
     903
     904          maxv = N.max(confusionmatrix)
     905          boundaries = N.linspace(0, maxv, N.min((maxv, 10)), True)
     906
     907          # Label axes
     908          P.xlabel("targets")
     909          P.ylabel("predictions")
     910
     911          P.setp(axi, xticks=ticks, yticks=yticks,
     912                 xticklabels=tick_labels, yticklabels=tick_labels)
     913
     914          axi.xaxis.set_ticks_position(xticks_position)
     915          axi.xaxis.set_label_position(xticks_position)
     916
     917          if xlabels_vertical:
     918              P.setp(P.getp(axi, 'xticklabels'), rotation='vertical')
     919
     920          axcb = fig.add_axes([0.8, ybottom, 0.02, 0.7])
     921          cb = P.colorbar(im, cax=axcb, format='%d', ticks = boundaries)
     922
     923          if P.matplotlib.get_backend() == 'TkAgg':
     924              P.ion()
     925          P.draw()
     926          # Store it primarily for testing
     927          self._plotted_confusionmatrix = confusionmatrix
     928          return fig, im, cb
     929
     930
     931      @property
     932 -    def error(self):
     933          self.compute()
     934          return 1.0-self.__Ncorrect*1.0/sum(self.__Nsamples)
     935
     936
     937      @property
     938 -    def labels(self):
     939          self.compute()
     940          return self.__labels
     941
     942
     943 -    def getLabels_map(self):
     944          return self.__labels_map
     945
     946
     947 -    def setLabels_map(self, val):
     948          if val is None or isinstance(val, dict):
     949              self.__labels_map = val
     950          else:
     951              raise ValueError, "Cannot set labels_map to %s" % val
     952          # reset it just in case
     953          self.__labels_map_rev = None
     954          self._computed = False
     955
     956
     957      @property
     958 -    def matrix(self):
     959          self.compute()
     960          return self.__matrix
     961
     962
     963      @property
     964 -    def percentCorrect(self):
     965          self.compute()
     966          return 100.0*self.__Ncorrect/sum(self.__Nsamples)
     967
     968      labels_map = property(fget=getLabels_map, fset=setLabels_map)
     969
     970
     971 -class RegressionStatistics(SummaryStatistics):
     972      """Class to contain information and display on regression results.
     973
     974      """
     975
     976      _STATS_DESCRIPTION = (
     977          ('CCe', 'Error based on correlation coefficient',
     978           '1 - corr_coef'),
     979          ('CCp', 'Correlation coefficient (p-value)', None),
     980          ('RMSE', 'Root mean squared error', None),
     981          ('STD', 'Standard deviation', None),
     982          ('RMP', 'Root mean power (compare to RMSE of results)',
     983           'sqrt(mean( data**2 ))'),
     984          ) + SummaryStatistics._STATS_DESCRIPTION
     985
     986
     987 -    def __init__(self, **kwargs):
     988          """Initialize RegressionStatistics
     989
     990          :Parameters:
     991           targets
     992             Optional set of targets
     993           predictions
     994             Optional set of predictions
     995             """
     996
     997          SummaryStatistics.__init__(self, **kwargs)
     998
     999
    1000 -    def _compute(self):
    1001          """Actually compute the confusion matrix based on all the sets"""
    1002
    1003          super(RegressionStatistics, self)._compute()
    1004          sets = self.sets
    1005          Nsets = len(sets)
    1006
    1007          stats = {}
    1008
    1009          funcs = {
    1010              'RMP_t': lambda p,t:rootMeanPowerFx(t),
    1011              'STD_t': lambda p,t:N.std(t),
    1012              'RMP_p': lambda p,t:rootMeanPowerFx(p),
    1013              'STD_p': lambda p,t:N.std(p),
    1014              'CCe': CorrErrorFx(),
    1015              'CCp': CorrErrorPFx(),
    1016              'RMSE': RMSErrorFx(),
    1017              'RMSE/RMP_t': RelativeRMSErrorFx()
    1018              }
    1019
    1020          for funcname, func in funcs.iteritems():
    1021              funcname_all = funcname + '_all'
    1022              stats[funcname_all] = []
    1023              for i, (targets, predictions, values) in enumerate(sets):
    1024                  stats[funcname_all] += [func(predictions, targets)]
    1025              stats[funcname_all] = N.array(stats[funcname_all])
    1026              stats[funcname] = N.mean(stats[funcname_all])
    1027              stats[funcname+'_std'] = N.std(stats[funcname_all])
    1028              stats[funcname+'_max'] = N.max(stats[funcname_all])
    1029              stats[funcname+'_min'] = N.min(stats[funcname_all])
    1030
    1031          # create ``summary`` statistics, since some per-set statistics
    1032          # might be uncomputable if a set contains just a single number
    1033          # (like in the case of correlation coefficient)
    1034          targets, predictions = [], []
    1035          for i, (targets_, predictions_, values_) in enumerate(sets):
    1036              targets += list(targets_)
    1037              predictions += list(predictions_)
    1038
    1039          for funcname, func in funcs.iteritems():
    1040              funcname_all = 'Summary ' + funcname
    1041              stats[funcname_all] = func(predictions, targets)
    1042
    1043          self._stats.update(stats)
    1044
    1045
    1046 -    def plot(self,
    1047               plot=True, plot_stats=True,
    1048               splot=True
    1049               #labels=None, numbers=False, origin='upper',
    1050               #numbers_alpha=None, xlabels_vertical=True,
    1051               #numbers_kwargs={},
    1052               #**kwargs
    1053               ):
    1054          """Provide presentation of regression performance in image
    1055
    1056          :Parameters:
    1057            plot : bool
    1058              Plot regular plot of values (targets/predictions)
    1059            plot_stats : bool
    1060              Print basic statistics in the title
    1061            splot : bool
    1062              Plot scatter plot
    1063
    1064          :Returns:
    1065             (fig, im, cb) -- figure, imshow, colorbar
    1066          """
    1067          externals.exists("pylab", raiseException=True)
    1068          import pylab as P
    1069
    1070          self.compute()
    1071          # total number of plots
    1072          nplots = plot + splot
    1073
    1074          # turn off automatic update if interactive
    1075          if P.matplotlib.get_backend() == 'TkAgg':
    1076              P.ioff()
    1077
    1078          fig = P.gcf()
    1079          P.clf()
    1080          sps = []                        # subplots
    1081
    1082          nplot = 0
    1083          if plot:
    1084              nplot += 1
    1085              sps.append(P.subplot(nplots, 1, nplot))
    1086              xstart = 0
    1087              lines = []
    1088              for s in self.sets:
    1089                  nsamples = len(s[0])
    1090                  xend = xstart+nsamples
    1091                  xs = xrange(xstart, xend)
    1092                  lines += [P.plot(xs, s[0], 'b')]
    1093                  lines += [P.plot(xs, s[1], 'r')]
    1094                  # vertical line
    1095                  P.plot([xend, xend], [N.min(s[0]), N.max(s[0])], 'k--')
    1096                  xstart = xend
    1097              if len(lines)>1:
    1098                  P.legend(lines[:2], ('Target', 'Prediction'))
    1099              if plot_stats:
    1100                  P.title(self.asstring(short='very'))
    1101
    1102          if splot:
    1103              nplot += 1
    1104              sps.append(P.subplot(nplots, 1, nplot))
    1105              for s in self.sets:
    1106                  P.plot(s[0], s[1], 'o',
    1107                         markeredgewidth=0.2,
    1108                         markersize=2)
    1109              P.gca().set_aspect('equal')
    1110
    1111          if P.matplotlib.get_backend() == 'TkAgg':
    1112              P.ion()
    1113          P.draw()
    1114
    1115          return fig, sps
    1116
    1117 -    def asstring(self, short=False, header=True,  summary=True,
    1118                   description=False):
    1119          """'Pretty print' the statistics"""
    1120
    1121          if len(self.sets) == 0:
    1122              return "Empty"
    1123
    1124          self.compute()
    1125
    1126          stats = self.stats
    1127
    1128          if short:
    1129              if short == 'very':
    1130                  # " RMSE/RMP_t:%(RMSE/RMP_t).2f" \
    1131                  return "%(# of sets)d sets CCe=%(CCe).2f p=%(CCp).2g" \
    1132                         " RMSE:%(RMSE).2f" \
    1133                         " Summary (stacked data): " \
    1134                         "CCe=%(Summary CCe).2f p=%(Summary CCp).2g" \
    1135                         % stats
    1136              else:
    1137                  return "%(# of sets)d sets CCe=%(CCe).2f+-%(CCe_std).3f" \
    1138                         " RMSE=%(RMSE).2f+-%(RMSE_std).3f" \
    1139                         " RMSE/RMP_t=%(RMSE/RMP_t).2f+-%(RMSE/RMP_t_std).3f" \
    1140                         % stats
    1141
    1142          stats_data = ['RMP_t', 'STD_t', 'RMP_p', 'STD_p']
    1143          # CCp needs tune up of format so excluded
    1144          stats_ = ['CCe', 'RMSE', 'RMSE/RMP_t']
    1145          stats_summary = ['# of sets']
    1146
    1147          out = StringIO()
    1148
    1149          printed = []
    1150          if header:
    1151              # labels
    1152              printed.append(['Statistics', 'Mean', 'Std', 'Min', 'Max'])
    1153              # underscores
    1154              printed.append(['----------', '-----', '-----', '-----', '-----'])
    1155
    1156          def print_stats(printed, stats_):
    1157              # Statistics itself
    1158              for stat in stats_:
    1159                  s = [stat]
    1160                  for suffix in ['', '_std', '_min', '_max']:
    1161                      s += [ _p2(stats[stat+suffix], 3) ]
    1162                  printed.append(s)
    1163
    1164          printed.append(["Data:     "])
    1165          print_stats(printed, stats_data)
    1166          printed.append(["Results:  "])
    1167          print_stats(printed, stats_)
    1168          printed.append(["Summary:  "])
    1169          printed.append(["CCe", _p2(stats['Summary CCe']), "", "p=", '%g' % stats['Summary CCp']])
    1170          printed.append(["RMSE", _p2(stats['Summary RMSE'])])
    1171          printed.append(["RMSE/RMP_t", _p2(stats['Summary RMSE/RMP_t'])])
    1172
    1173          if summary:
    1174              for stat in stats_summary:
    1175                  printed.append([stat] + [_p2(stats[stat])])
    1176
    1177          table2string(printed, out)
    1178
    1179          if description:
    1180              out.write("\nDescription of printed statistics.\n"
    1181                        " Suffixes: _t - targets, _p - predictions\n")
    1182
    1183              for d, val, eq in self._STATS_DESCRIPTION:
    1184                  out.write(" %-3s: %s\n" % (d, val))
    1185                  if eq is not None:
    1186                      out.write("      " + eq + "\n")
    1187
    1188          result = out.getvalue()
    1189          out.close()
    1190          return result
    1191
    1192
    1193      @property
    1194 -    def error(self):
    1195          self.compute()
    1196          return self.stats['RMSE']
    1197
    1198
    1199
    1200 -class ClassifierError(ClassWithCollections):
    1201      """Compute (or return) some error of a (trained) classifier on a dataset.
    1202      """
    1203
    1204      confusion = StateVariable(enabled=False)
    1205      """TODO Think that labels might be also symbolic thus can't directly
    1206         be indicies of the array
    1207      """
    1208
    1209      training_confusion = StateVariable(enabled=False,
    1210          doc="Proxy training_confusion from underlying classifier.")
    1211
    1212
    1213 -    def __init__(self, clf, labels=None, train=True, **kwargs):
    1214          """Initialization.
    1215
    1216          :Parameters:
    1217            clf : Classifier
    1218              Either trained or untrained classifier
    1219            labels : list
    1220              if provided, should be a set of labels to add on top of the
    1221              ones present in testdata
    1222            train : bool
    1223              unless train=False, classifier gets trained if
    1224              trainingdata provided to __call__
    1225          """
    1226          ClassWithCollections.__init__(self, **kwargs)
    1227          self.__clf = clf
    1228
    1229          self._labels = labels
    1230          """Labels to add on top to existing in testing data"""
    1231
    1232          self.__train = train
    1233          """Either to train classifier if trainingdata is provided"""
    1234
    1235
    1236      __doc__ = enhancedDocString('ClassifierError', locals(), ClassWithCollections)
    1237
    1238
    1239 -    def __copy__(self):
    1240          """TODO: think... may be we need to copy self.clf"""
    1241          out = ClassifierError.__new__(TransferError)
    1242          ClassifierError.__init__(out, self.clf)
    1243          return out
    1244
    1245
    1246 -    def _precall(self, testdataset, trainingdataset=None):
    1247          """Generic part which trains the classifier if necessary
    1248          """
    1249          if not trainingdataset is None:
    1250              if self.__train:
    1251                  # XXX can be pretty annoying if triggered inside an algorithm
    1252                  # where it cannot be switched of, but retraining might be
    1253                  # intended or at least not avoidable.
    1254                  # Additonally isTrained docs say:
    1255                  #   MUST BE USED WITH CARE IF EVER
    1256                  #
    1257                  # switching it off for now
    1258                  #if self.__clf.isTrained(trainingdataset):
    1259                  #    warning('It seems that classifier %s was already trained' %
    1260                  #            self.__clf + ' on dataset %s. Please inspect' \
    1261                  #                % trainingdataset)
    1262                  if self.states.isEnabled('training_confusion'):
    1263                      self.__clf.states._changeTemporarily(
    1264                          enable_states=['training_confusion'])
    1265                  self.__clf.train(trainingdataset)
    1266                  if self.states.isEnabled('training_confusion'):
    1267                      self.training_confusion = self.__clf.training_confusion
    1268                      self.__clf.states._resetEnabledTemporarily()
    1269
    1270          if self.__clf.states.isEnabled('trained_labels') and \
    1271                 not testdataset is None:
    1272              newlabels = set(testdataset.uniquelabels) \
    1273                          - set(self.__clf.trained_labels)
    1274              if len(newlabels)>0:
    1275                  warning("Classifier %s wasn't trained to classify labels %s" %
    1276                          (`self.__clf`, `newlabels`) +
    1277                          " present in testing dataset. Make sure that you have" +
    1278                          " not mixed order/names of the arguments anywhere")
    1279
    1280          ### Here checking for if it was trained... might be a cause of trouble
    1281          # XXX disabled since it is unreliable.. just rely on explicit
    1282          # self.__train
    1283          #    if  not self.__clf.isTrained(trainingdataset):
    1284          #        self.__clf.train(trainingdataset)
    1285          #    elif __debug__:
    1286          #        debug('CERR',
    1287          #              'Not training classifier %s since it was ' % `self.__clf`
    1288          #              + ' already trained on dataset %s' % `trainingdataset`)
    1289
    1290
    1291 -    def _call(self, testdataset, trainingdataset=None):
    1292          raise NotImplementedError
    1293
    1294
    1295 -    def _postcall(self, testdataset, trainingdataset=None, error=None):
    1296          pass
    1297
    1298
    1299 -    def __call__(self, testdataset, trainingdataset=None):
    1300          """Compute the transfer error for a certain test dataset.
    1301
    1302          If `trainingdataset` is not `None` the classifier is trained using the
    1303          provided dataset before computing the transfer error. Otherwise the
    1304          classifier is used in it's current state to make the predictions on
    1305          the test dataset.
    1306
    1307          Returns a scalar value of the transfer error.
    1308          """
    1309          self._precall(testdataset, trainingdataset)
    1310          error = self._call(testdataset, trainingdataset)
    1311          self._postcall(testdataset, trainingdataset, error)
    1312          if __debug__:
    1313              debug('CERR', 'Classifier error on %s: %.2f'
    1314                    % (testdataset, error))
    1315          return error
    1316
    1317
    1318 -    def untrain(self):
    1319          """Untrain the *Error which relies on the classifier
    1320          """
    1321          self.clf.untrain()
    1322
    1323
    1324      @property
    1325 -    def clf(self):
    1326          return self.__clf
    1327
    1328
    1329      @property
    1330 -    def labels(self):
    1331          return self._labels
    1332
    1333
    1334
    1335 -class TransferError(ClassifierError):
    1336      """Compute the transfer error of a (trained) classifier on a dataset.
    1337
    1338      The actual error value is computed using a customizable error function.
    1339      Optionally the classifier can be trained by passing an additional
    1340      training dataset to the __call__() method.
    1341      """
    1342
    1343      null_prob = StateVariable(enabled=True,
    1344                      doc="Stores the probability of an error result under "
    1345                           "the NULL hypothesis")
    1346      samples_error = StateVariable(enabled=False,
    1347                          doc="Per sample errors computed by invoking the "
    1348                              "error function for each sample individually. "
    1349                              "Errors are available in a dictionary with each "
    1350                              "samples origid as key.")
    1351
    1352 -    def __init__(self, clf, errorfx=MeanMismatchErrorFx(), labels=None,
    1353                   null_dist=None, **kwargs):
    1354          """Initialization.
    1355
    1356          :Parameters:
    1357            clf : Classifier
    1358              Either trained or untrained classifier
    1359            errorfx
    1360              Functor that computes a scalar error value from the vectors of
    1361              desired and predicted values (e.g. subclass of `ErrorFunction`)
    1362            labels : list
    1363              if provided, should be a set of labels to add on top of the
    1364              ones present in testdata
    1365            null_dist : instance of distribution estimator
    1366          """
    1367          ClassifierError.__init__(self, clf, labels, **kwargs)
    1368          self.__errorfx = errorfx
    1369          self.__null_dist = autoNullDist(null_dist)
    1370
    1371
    1372      __doc__ = enhancedDocString('TransferError', locals(), ClassifierError)
    1373
    1374
    1375 -    def __copy__(self):
    1376          """Performs deepcopying of the classifier."""
    1377          # TODO -- use ClassifierError.__copy__
    1378          out = TransferError.__new__(TransferError)
    1379          TransferError.__init__(out, self.clf.clone(), self.errorfx, self._labels)
    1380
    1381          return out
    1382
    1383      # XXX: TODO: unify naming? test/train or with ing both
    1384 -    def _call(self, testdataset, trainingdataset=None):
    1385          """Compute the transfer error for a certain test dataset.
    1386
    1387          If `trainingdataset` is not `None` the classifier is trained using the
    1388          provided dataset before computing the transfer error. Otherwise the
    1389          classifier is used in it's current state to make the predictions on
    1390          the test dataset.
    1391
    1392          Returns a scalar value of the transfer error.
    1393          """
    1394          # OPT: local binding
    1395          clf = self.clf
    1396          if testdataset is None:
    1397              # We cannot do anythin, but we can try to figure out WTF and
    1398              # warn the user accordingly in some usecases
    1399              import traceback as tb
    1400              filenames = [x[0] for x in tb.extract_stack(limit=100)]
    1401              rfe_matches = [f for f in filenames if f.endswith('/rfe.py')]
    1402              cv_matches = [f for f in filenames if
    1403                            f.endswith('cvtranserror.py')]
    1404              msg = ""
    1405              if len(rfe_matches) > 0 and len(cv_matches):
    1406                  msg = " It is possible that you used RFE with stopping " \
    1407                        "criterion based on the TransferError and directly" \
    1408                        " from CrossValidatedTransferError, such approach" \
    1409                        " would require exposing testing dataset " \
    1410                        " to the classifier which might heavily bias " \
    1411                        " generalization performance estimate. If you are " \
    1412                        " sure to use it that way, create CVTE with " \
    1413                        " parameter expose_testdataset=True"
    1414              raise ValueError, "Transfer error call obtained None " \
    1415                    "as a dataset for testing.%s" % msg
    1416          predictions = clf.predict(testdataset.samples)
    1417
    1418          # compute confusion matrix
    1419          # Should it migrate into ClassifierError.__postcall?
    1420          # -> Probably not because other childs could estimate it
    1421          #  not from test/train datasets explicitely, see
    1422          #  `ConfusionBasedError`, where confusion is simply
    1423          #  bound to classifiers confusion matrix
    1424          states = self.states
    1425          if states.isEnabled('confusion'):
    1426              confusion = clf._summaryClass(
    1427                  #labels=self.labels,
    1428                  targets=testdataset.labels,
    1429                  predictions=predictions,
    1430                  values=clf.states.get('values', None))
    1431              try:
    1432                  confusion.labels_map = testdataset.labels_map
    1433              except:
    1434                  pass
    1435              states.confusion = confusion
    1436
    1437          if states.isEnabled('samples_error'):
    1438              samples_error = []
    1439              for i, p in enumerate(predictions):
    1440                  samples_error.append(self.__errorfx([p], testdataset.labels[i:i+1]))
    1441
    1442              states.samples_error = dict(zip(testdataset.origids, samples_error))
    1443
    1444          # compute error from desired and predicted values
    1445          error = self.__errorfx(predictions, testdataset.labels)
    1446
    1447          return error
    1448
    1449
    1450 -    def _postcall(self, vdata, wdata=None, error=None):
    1451          """
    1452          """
    1453          # estimate the NULL distribution when functor and training data is
    1454          # given
    1455          if not self.__null_dist is None and not wdata is None:
    1456              # we need a matching transfer error instances (e.g. same error
    1457              # function), but we have to disable the estimation of the null
    1458              # distribution in that child to prevent infinite looping.
    1459              null_terr = copy.copy(self)
    1460              null_terr.__null_dist = None
    1461              self.__null_dist.fit(null_terr, wdata, vdata)
    1462
    1463
    1464          # get probability of error under NULL hypothesis if available
    1465          if not error is None and not self.__null_dist is None:
    1466              self.null_prob = self.__null_dist.p(error)
    1467
    1468
    1469      @property
    1470 -    def errorfx(self): return self.__errorfx
    1471
    1472      @property
    1473 -    def null_dist(self): return self.__null_dist
    1474
    1475
    1476
    1477 -class ConfusionBasedError(ClassifierError):
    1478      """For a given classifier report an error based on internally
    1479      computed error measure (given by some `ConfusionMatrix` stored in
    1480      some state variable of `Classifier`).
    1481
    1482      This way we can perform feature selection taking as the error
    1483      criterion either learning error, or transfer to splits error in
    1484      the case of SplitClassifier
    1485      """
    1486
    1487 -    def __init__(self, clf, labels=None, confusion_state="training_confusion",
    1488                   **kwargs):
    1489          """Initialization.
    1490
    1491          :Parameters:
    1492            clf : Classifier
    1493              Either trained or untrained classifier
    1494            confusion_state
    1495              Id of the state variable which stores `ConfusionMatrix`
    1496            labels : list
    1497              if provided, should be a set of labels to add on top of the
    1498              ones present in testdata
    1499          """
    1500          ClassifierError.__init__(self, clf, labels, **kwargs)
    1501
    1502          self.__confusion_state = confusion_state
    1503          """What state to extract from"""
    1504
    1505          if not clf.states.isKnown(confusion_state):
    1506              raise ValueError, \
    1507                    "State variable %s is not defined for classifier %s" % \
    1508                    (confusion_state, `clf`)
    1509          if not clf.states.isEnabled(confusion_state):
    1510              if __debug__:
    1511                  debug('CERR', "Forcing state %s to be enabled for %s" %
    1512                        (confusion_state, `clf`))
    1513              clf.states.enable(confusion_state)
    1514
    1515
    1516      __doc__ = enhancedDocString('ConfusionBasedError', locals(),
    1517                                  ClassifierError)
    1518
    1519
    1520 -    def _call(self, testdata, trainingdata=None):
    1521          """Extract transfer error. Nor testdata, neither trainingdata is used
    1522          """
    1523          confusion = self.clf.states[self.__confusion_state].value
    1524          self.confusion = confusion
    1525          return confusion.error
    1526