Package mvpa :: Package misc :: Module state

[hide private]
[frames] | no frames]

# Source Code for Module mvpa.misc.state

       1  # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
       2  # vi: set ft=python sts=4 ts=4 sw=4 et:
       3  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
       4  #
       5  #   See COPYING file distributed along with the PyMVPA package for the
       6  #   copyright and license terms.
       7  #
       8  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
       9  """Classes to control and store state information.
      10
      11  It was devised to provide conditional storage
      12  """
      13
      14  # XXX: MH: The use of `index` as variable name confuses me. IMHO `index` refers
      15  #          to a position in a container (i.e. list access). However, in this
      16  #          file it is mostly used in the context of a `key` for dictionary
      17  #          access. Can we refactor that?
      18  __docformat__ = 'restructuredtext'
      19
      20  import operator, copy
      21  from textwrap import TextWrapper
      22
      23  import numpy as N
      24
      25  from mvpa.misc.vproperty import VProperty
      26  from mvpa.misc.exceptions import UnknownStateError
      27  from mvpa.misc.attributes import CollectableAttribute, StateVariable
      28  from mvpa.base.dochelpers import enhancedDocString
      29
      30  from mvpa.base import externals
      31
      32  if __debug__:
      33      from mvpa.base import debug
      34
      35
      36  _in_ipython = externals.exists('running ipython env')
      37  # Separators around definitions, needed for ReST, but bogus for
      38  # interactive sessions
      39  _def_sep = ('`', '')[int(_in_ipython)]
      40
      41  _object_getattribute = object.__getattribute__
      42  _object_setattr = object.__setattr__
      43
      44
      45  ###################################################################
      46  # Collections
      47  #
      48  # TODO: refactor into collections.py. state.py now has
      49  #       little in common with the main part of this file
      50  #
      51 -class Collection(object):
      52      """Container of some CollectableAttributes.
      53
      54      :Groups:
      55       - `Public Access Functions`: `isKnown`
      56       - `Access Implementors`: `_getListing`, `_getNames`
      57       - `Mutators`: `__init__`
      58       - `R/O Properties`: `listing`, `names`, `items`
      59
      60       XXX Seems to be not used and duplicating functionality: `_getListing`
      61       (thus `listing` property)
      62      """
      63
      64 -    def __init__(self, items=None, owner=None, name=None):
      65          """Initialize the Collection
      66
      67          :Parameters:
      68            items : dict of CollectableAttribute's
      69              items to initialize with
      70            owner : object
      71              an object to which collection belongs
      72            name : basestring
      73              name of the collection (as seen in the owner, e.g. 'states')
      74          """
      75
      76          self.__owner = owner
      77
      78          if items == None:
      79              items = {}
      80          self._items = items
      81          """Dictionary to contain registered states as keys and
      82          values signal either they are enabled
      83          """
      84          self.__name = name
      85
      86 -    def _setName(self, name):
      87          self.__name = name
      88
      89 -    def __str__(self):
      90          num = len(self._items)
      91          if __debug__ and "ST" in debug.active:
      92              maxnumber = 1000            # I guess all
      93          else:
      94              maxnumber = 4
      95          if self.__name is not None:
      96              res = self.__name
      97          else:
      98              res = ""
      99          res += "{"
     100          for i in xrange(min(num, maxnumber)):
     101              if i > 0:
     102                  res += " "
     103              res += "%s" % str(self._items.values()[i])
     104          if len(self._items) > maxnumber:
     105              res += "..."
     106          res += "}"
     107          if __debug__:
     108              if "ST" in debug.active:
     109                  res += " owner:%s#%s" % (self.owner.__class__.__name__,
     110                                           id(self.owner))
     111          return res
     112
     113
     114 -    def _cls_repr(self):
     115          """Collection specific part of __repr__ for a class containing
     116          it, ie a part of __repr__ for the owner object
     117
     118          :Return:
     119            list of items to be appended within __repr__ after a .join()
     120          """
     121          # XXX For now we do not expect any pure non-specialized
     122          # collection , thus just override in derived classes
     123          raise NotImplementedError, "Class %s should override _cls_repr" \
     124                % self.__class__.__name__
     125
     126 -    def _is_initializable(self, index):
     127          """Checks if index could be assigned within collection via
     128          _initialize
     129
     130          :Return: bool value for a given `index`
     131
     132          It is to facilitate dynamic assignment of collections' items
     133          within derived classes' __init__ depending on the present
     134          collections in the class.
     135          """
     136          # XXX Each collection has to provide what indexes it allows
     137          #     to be set within constructor. Custom handling of some
     138          #     arguments (like (dis|en)able_states) is to be performed
     139          #     in _initialize
     140          # raise NotImplementedError, \
     141          #      "Class %s should override _is_initializable" \
     142          #      % self.__class__.__name__
     143
     144          # YYY lets just check if it is in the keys
     145          return index in self._items.keys()
     146
     147
     148 -    def _initialize(self, index, value):
     149          """Initialize `index` (no check performed) with `value`
     150          """
     151          # by default we just set corresponding value
     152          self[index].value = value
     153
     154
     155 -    def __repr__(self):
     156          s = "%s(" % self.__class__.__name__
     157          items_s = ""
     158          sep = ""
     159          for item in self._items:
     160              try:
     161                  itemvalue = "%s" % `self._items[item].value`
     162                  if len(itemvalue)>50:
     163                      itemvalue = itemvalue[:10] + '...' + itemvalue[-10:]
     164                  items_s += "%s'%s':%s" % (sep, item, itemvalue)
     165                  sep = ', '
     166              except:
     167                  pass
     168          if items_s != "":
     169              s += "items={%s}" % items_s
     170          if self.owner is not None:
     171              s += "%sowner=%s" % (sep, `self.owner`)
     172          s += ")"
     173          return s
     174
     175
     176      #
     177      # XXX TODO: figure out if there is a way to define proper
     178      #           __copy__'s for a hierarchy of classes. Probably we had
     179      #           to define __getinitargs__, etc... read more...
     180      #
     181      #def __copy__(self):
     182  # TODO Remove or refactor?
     183  #    def _copy_states_(self, fromstate, deep=False):
     184  #        """Copy known here states from `fromstate` object into current object
     185  #
     186  #        Crafted to overcome a problem mentioned above in the comment
     187  #        and is to be called from __copy__ of derived classes
     188  #
     189  #        Probably sooner than later will get proper __getstate__,
     190  #        __setstate__
     191  #        """
     192  #        # Bad check... doesn't generalize well...
     193  #        # if not issubclass(fromstate.__class__, self.__class__):
     194  #        #     raise ValueError, \
     195  #        #           "Class  %s is not subclass of %s, " % \
     196  #        #           (fromstate.__class__, self.__class__) + \
     197  #        #           "thus not eligible for _copy_states_"
     198  #        # TODO: FOR NOW NO TEST! But this beast needs to be fixed...
     199  #        operation = { True: copy.deepcopy,
     200  #                      False: copy.copy }[deep]
     201  #
     202  #        if isinstance(fromstate, ClassWithCollections):
     203  #            fromstate = fromstate.states
     204  #
     205  #        self.enabled = fromstate.enabled
     206  #        for name in self.names:
     207  #            if fromstate.isKnown(name):
     208  #                self._items[name] = operation(fromstate._items[name])
     209
     210
     211 -    def isKnown(self, index):
     212          """Returns `True` if state `index` is known at all"""
     213          return self._items.has_key(index)
     214
     215
     216 -    def isSet(self, index=None):
     217          """If item (or any in the present or listed) was set
     218
     219          :Parameters:
     220            index : None or basestring or list of basestring
     221              What items to check if they were set in the collection
     222          """
     223          _items = self._items
     224          if not (index is None):
     225              if isinstance(index, basestring):
     226                   self._checkIndex(index) # process just that single index
     227                   return _items[index].isSet
     228              else:
     229                  items = index           # assume that we got some list
     230          else:
     231              items = self._items         # go through all the items
     232
     233          for index in items:
     234              self._checkIndex(index)
     235              if _items[index].isSet:
     236                  return True
     237          return False
     238
     239
     240 -    def whichSet(self):
     241          """Return list of indexes which were set"""
     242          result = []
     243          # go through all members and if any isSet -- return True
     244          for index,v in self._items.iteritems():
     245              if v.isSet:
     246                  result.append(index)
     247          return result
     248
     249
     250 -    def _checkIndex(self, index):
     251          """Verify that given `index` is a known/registered state.
     252
     253          :Raise `KeyError`: if given `index` is not known
     254          """
     255          # OPT: lets not reuse isKnown, to don't incure 1 more function
     256          #      call
     257          if not self._items.has_key(index):
     258              raise KeyError, \
     259                    "%s of %s has no key '%s' registered" \
     260                    % (self.__class__.__name__,
     261                       self.__owner.__class__.__name__,
     262                       index)
     263
     264
     265 -    def add(self, item):
     266          """Add a new CollectableAttribute to the collection
     267
     268          :Parameters:
     269            item : CollectableAttribute
     270              or of derived class. Must have 'name' assigned
     271
     272          TODO: we should make it stricter to don't add smth of
     273                wrong type into Collection since it might lead to problems
     274
     275                Also we might convert to __setitem__
     276          """
     277          # local binding
     278          name = item.name
     279          if not isinstance(item, CollectableAttribute):
     280              raise ValueError, \
     281                    "Collection can add only instances of " + \
     282                    "CollectableAttribute-derived classes. Got %s" % `item`
     283
     284          if name is None:
     285              raise ValueError, \
     286                    "CollectableAttribute to be added %s must have 'name' set" % \
     287                    item
     288          self._items[name] = item
     289
     290          if not self.owner is None:
     291              self._updateOwner(name)
     292
     293
     294 -    def remove(self, index):
     295          """Remove item from the collection
     296          """
     297          self._checkIndex(index)
     298          self._updateOwner(index, register=False)
     299          discard = self._items.pop(index)
     300
     301
     302 -    def __getattribute__(self, index):
     303          """
     304          """
     305          #return all private and protected ones first since we will not have
     306          # collectable's with _ (we should not have!)
     307          if index[0] == '_':
     308              return _object_getattribute(self, index)
     309          _items = _object_getattribute(self, '_items')
     310          if index in _items:
     311              return _items[index].value
     312          return _object_getattribute(self, index)
     313
     314
     315 -    def __setattr__(self, index, value):
     316          if index[0] == '_':
     317              return _object_setattr(self, index, value)
     318          _items = _object_getattribute(self, '_items')
     319          if index in _items:
     320              _items[index].value = value
     321          else:
     322              _object_setattr(self, index, value)
     323
     324
     325 -    def __getitem__(self, index):
     326          _items = _object_getattribute(self, '_items')
     327          if index in _items:
     328              self._checkIndex(index)
     329              return _items[index]
     330          else:
     331              raise AttributeError("State collection %s has no %s attribute"
     332                                   % (self, index))
     333
     334
     335      # Probably not needed -- enable if need arises
     336      #
     337      #def __setattr__(self, index, value):
     338      #    if self._items.has_key(index):
     339      #        self._updateOwner(index, register=False)
     340      #        self._items[index] = value
     341      #        self._updateOwner(index, register=True)
     342      #
     343      #    _object_setattr(self, index, value)
     344
     345
     346 -    def get(self, index, default):
     347          """Access the value by a given index.
     348
     349          Mimiquing regular dictionary behavior, if value cannot be obtained
     350          (i.e. if any exception is caught) return default value.
     351          """
     352          try:
     353              return self[index].value
     354          except Exception, e:
     355              #if default is not None:
     356              return default
     357              #else:
     358              #    raise e
     359
     360
     361 -    def _action(self, index, func, missingok=False, **kwargs):
     362          """Run specific func either on a single item or on all of them
     363
     364          :Parameters:
     365            index : basestr
     366              Name of the state variable
     367            func
     368              Function (not bound) to call given an item, and **kwargs
     369            missingok : bool
     370              If True - do not complain about wrong index
     371          """
     372          if isinstance(index, basestring):
     373              if index.upper() == 'ALL':
     374                  for index_ in self._items:
     375                      self._action(index_, func, missingok=missingok, **kwargs)
     376              else:
     377                  try:
     378                      self._checkIndex(index)
     379                      func(self._items[index], **kwargs)
     380                  except:
     381                      if missingok:
     382                          return
     383                      raise
     384          elif operator.isSequenceType(index):
     385              for item in index:
     386                  self._action(item, func, missingok=missingok, **kwargs)
     387          else:
     388              raise ValueError, \
     389                    "Don't know how to handle  variable given by %s" % index
     390
     391
     392 -    def reset(self, index=None):
     393          """Reset the state variable defined by `index`"""
     394
     395          if not index is None:
     396              indexes = [ index ]
     397          else:
     398              indexes = self.names
     399
     400          if len(self.items):
     401              for index in indexes:
     402                  # XXX Check if that works as desired
     403                  self._action(index, self._items.values()[0].__class__.reset,
     404                               missingok=False)
     405
     406
     407 -    def _getListing(self):
     408          """Return a list of registered states along with the documentation"""
     409
     410          # lets assure consistent litsting order
     411          items = self._items.items()
     412          items.sort()
     413          return [ "%s%s%s: %s" % (_def_sep, str(x[1]), _def_sep, x[1].__doc__)
     414                   for x in items ]
     415
     416
     417 -    def _getNames(self):
     418          """Return ids for all registered state variables"""
     419          return self._items.keys()
     420
     421
     422 -    def _getOwner(self):
     423          return self.__owner
     424
     425
     426 -    def _setOwner(self, owner):
     427          if not isinstance(owner, ClassWithCollections):
     428              raise ValueError, \
     429                    "Owner of the StateCollection must be ClassWithCollections object"
     430          if __debug__:
     431              try:    strowner = str(owner)
     432              except: strowner = "UNDEF: <%s#%s>" % (owner.__class__, id(owner))
     433              debug("ST", "Setting owner for %s to be %s" % (self, strowner))
     434          if not self.__owner is None:
     435              # Remove attributes which were registered to that owner previousely
     436              self._updateOwner(register=False)
     437          self.__owner = owner
     438          if not self.__owner is None:
     439              self._updateOwner(register=True)
     440
     441
     442 -    def _updateOwner(self, index=None, register=True):
     443          """Define an entry within owner's __dict__
     444           so ipython could easily complete it
     445
     446           :Parameters:
     447             index : basestring or list of basestring
     448               Name of the attribute. If None -- all known get registered
     449             register : bool
     450               Register if True or unregister if False
     451
     452           XXX Needs refactoring since we duplicate the logic of expansion of
     453           index value
     454          """
     455          if not index is None:
     456              if not index in self._items:
     457                  raise ValueError, \
     458                        "Attribute %s is not known to %s" % (index, self)
     459              indexes = [ index ]
     460          else:
     461              indexes = self.names
     462
     463          ownerdict = self.owner.__dict__
     464          selfdict = self.__dict__
     465          owner_known = ownerdict['_known_attribs']
     466          for index_ in indexes:
     467              if register:
     468                  if index_ in ownerdict:
     469                      raise RuntimeError, \
     470                            "Cannot register attribute %s within %s " % \
     471                            (index_, self.owner) + "since it has one already"
     472                  ownerdict[index_] = self._items[index_]
     473                  if index_ in selfdict:
     474                      raise RuntimeError, \
     475                            "Cannot register attribute %s within %s " % \
     476                            (index_, self) + "since it has one already"
     477                  selfdict[index_] = self._items[index_]
     478                  owner_known[index_] = self.__name
     479              else:
     480                  if index_ in ownerdict:
     481                      # yoh doesn't think that we need to complain if False
     482                      ownerdict.pop(index_)
     483                      owner_known.pop(index_)
     484                  if index_ in selfdict:
     485                      selfdict.pop(index_)
     486
     487
     488      # Properties
     489      names = property(fget=_getNames)
     490      items = property(fget=lambda x:x._items)
     491      owner = property(fget=_getOwner, fset=_setOwner)
     492      name = property(fget=lambda x:x.__name, fset=_setName)
     493
     494      # Virtual properties
     495      listing = VProperty(fget=_getListing)
     496
     497
     498
     499 -class ParameterCollection(Collection):
     500      """Container of Parameters for a stateful object.
     501      """
     502
     503  #    def __init__(self, items=None, owner=None, name=None):
     504  #        """Initialize the state variables of a derived class
     505  #
     506  #        :Parameters:
     507  #          items : dict
     508  #            dictionary of states
     509  #        """
     510  #        Collection.__init__(self, items, owner, name)
     511  #
     512
     513 -    def _cls_repr(self):
     514          """Part of __repr__ for the owner object
     515          """
     516          prefixes = []
     517          for k in self.names:
     518              # list only params with not default values
     519              if self[k].isDefault:
     520                  continue
     521              prefixes.append("%s=%s" % (k, self[k].value))
     522          return prefixes
     523
     524
     525 -    def resetvalue(self, index, missingok=False):
     526          """Reset all parameters to default values"""
     527          from param import Parameter
     528          self._action(index, Parameter.resetvalue, missingok=False)
     529
     530
     531 -class SampleAttributesCollection(Collection):
     532      """Container for data and attributes of samples (ie data/labels/chunks/...)
     533      """
     534
     535  #    def __init__(self, items=None, owner=None, name=None):
     536  #        """Initialize the state variables of a derived class
     537  #
     538  #        :Parameters:
     539  #          items : dict
     540  #            dictionary of states
     541  #        """
     542  #        Collection.__init__(self, items, owner, name)
     543  #
     544
     545 -    def _cls_repr(self):
     546          """Part of __repr__ for the owner object
     547          """
     548          return [] # TODO: return I guess samples/labels/chunks
     549
     550
     551
     552 -class StateCollection(Collection):
     553      """Container of StateVariables for a stateful object.
     554
     555      :Groups:
     556       - `Public Access Functions`: `isKnown`, `isEnabled`, `isActive`
     557       - `Access Implementors`: `_getListing`, `_getNames`, `_getEnabled`
     558       - `Mutators`: `__init__`, `enable`, `disable`, `_setEnabled`
     559       - `R/O Properties`: `listing`, `names`, `items`
     560       - `R/W Properties`: `enabled`
     561      """
     562
     563 -    def __init__(self, items=None, owner=None):
     564          """Initialize the state variables of a derived class
     565
     566          :Parameters:
     567            items : dict
     568              dictionary of states
     569            owner : ClassWithCollections
     570              object which owns the collection
     571            name : basestring
     572              literal description. Usually just attribute name for the
     573              collection, e.g. 'states'
     574          """
     575          Collection.__init__(self, items=items, owner=owner)
     576
     577          self.__storedTemporarily = []
     578          """List to contain sets of enabled states which were enabled
     579          temporarily.
     580          """
     581
     582      #
     583      # XXX TODO: figure out if there is a way to define proper
     584      #           __copy__'s for a hierarchy of classes. Probably we had
     585      #           to define __getinitargs__, etc... read more...
     586      #
     587      #def __copy__(self):
     588
     589 -    def _cls_repr(self):
     590          """Part of __repr__ for the owner object
     591          """
     592          prefixes = []
     593          for name, invert in ( ('enable', False), ('disable', True) ):
     594              states = self._getEnabled(nondefault=False,
     595                                        invert=invert)
     596              if len(states):
     597                  prefixes.append("%s_states=%s" % (name, str(states)))
     598          return prefixes
     599
     600
     601 -    def _is_initializable(self, index):
     602          """Checks if index could be assigned within collection via
     603          setvalue
     604          """
     605          return index in ['enable_states', 'disable_states']
     606
     607
     608 -    def _initialize(self, index, value):
     609          if value is None:
     610              value = []
     611          if index == 'enable_states':
     612              self.enable(value, missingok=True)
     613          elif index == 'disable_states':
     614              self.disable(value)
     615          else:
     616              raise ValueError, "StateCollection can accept only enable_states " \
     617                    "and disable_states arguments for the initialization. " \
     618                    "Got %s" % index
     619
     620
     621 -    def _copy_states_(self, fromstate, index=None, deep=False):
     622          """Copy known here states from `fromstate` object into current object
     623
     624          :Parameters:
     625            fromstate : Collection or ClassWithCollections
     626              Source states to copy from
     627            index : None or list of basestring
     628              If not to copy all set state variables, index provides
     629              selection of what to copy
     630            deep : bool
     631              Optional control over the way to copy
     632
     633          Crafted to overcome a problem mentioned above in the comment
     634          and is to be called from __copy__ of derived classes
     635
     636          Probably sooner than later will get proper __getstate__,
     637          __setstate__
     638          """
     639          # Bad check... doesn't generalize well...
     640          # if not issubclass(fromstate.__class__, self.__class__):
     641          #     raise ValueError, \
     642          #           "Class  %s is not subclass of %s, " % \
     643          #           (fromstate.__class__, self.__class__) + \
     644          #           "thus not eligible for _copy_states_"
     645          # TODO: FOR NOW NO TEST! But this beast needs to be fixed...
     646          operation = { True: copy.deepcopy,
     647                        False: copy.copy }[deep]
     648
     649          if isinstance(fromstate, ClassWithCollections):
     650              fromstate = fromstate.states
     651
     652          #self.enabled = fromstate.enabled
     653          _items, from_items = self._items, fromstate._items
     654          if index is None:
     655              # copy all set ones
     656              for name in fromstate.whichSet():#self.names:
     657                  #if fromstate.isKnown(name):
     658                  _items[name] = operation(from_items[name])
     659          else:
     660              isKnown = fromstate.isKnown
     661              for name in index:
     662                  if isKnown(name):
     663                      _items[name] = operation(from_items[name])
     664
     665
     666 -    def isEnabled(self, index):
     667          """Returns `True` if state `index` is enabled"""
     668          self._checkIndex(index)
     669          return self._items[index].isEnabled
     670
     671
     672 -    def isActive(self, index):
     673          """Returns `True` if state `index` is known and is enabled"""
     674          return self.isKnown(index) and self.isEnabled(index)
     675
     676
     677 -    def enable(self, index, value=True, missingok=False):
     678          """Enable  state variable given in `index`"""
     679          self._action(index, StateVariable.enable, missingok=missingok,
     680                       value=value)
     681
     682
     683 -    def disable(self, index):
     684          """Disable state variable defined by `index` id"""
     685          self._action(index, StateVariable.enable, missingok=False, value=False)
     686
     687
     688      # TODO XXX think about some more generic way to grab temporary
     689      # snapshot of CollectableAttributes to be restored later on...
     690 -    def _changeTemporarily(self, enable_states=None,
     691                             disable_states=None, other=None):
     692          """Temporarily enable/disable needed states for computation
     693
     694          Enable or disable states which are enabled in `other` and listed in
     695          `enable _states`. Use `resetEnabledTemporarily` to reset
     696          to previous state of enabled.
     697
     698          `other` can be a ClassWithCollections object or StateCollection
     699          """
     700          if enable_states == None:
     701              enable_states = []
     702          if disable_states == None:
     703              disable_states = []
     704          self.__storedTemporarily.append(self.enabled)
     705          other_ = other
     706          if isinstance(other, ClassWithCollections):
     707              other = other.states
     708
     709          if not other is None:
     710              # lets take states which are enabled in other but not in
     711              # self
     712              add_enable_states = list(set(other.enabled).difference(
     713                   set(enable_states)).intersection(self.names))
     714              if len(add_enable_states)>0:
     715                  if __debug__:
     716                      debug("ST",
     717                            "Adding states %s from %s to be enabled temporarily" %
     718                            (add_enable_states, other_) +
     719                            " since they are not enabled in %s" %
     720                            (self))
     721                  enable_states += add_enable_states
     722
     723          # Lets go one by one enabling only disabled once... but could be as
     724          # simple as
     725          self.enable(enable_states)
     726          self.disable(disable_states)
     727
     728
     729 -    def _resetEnabledTemporarily(self):
     730          """Reset to previousely stored set of enabled states"""
     731          if __debug__:
     732              debug("ST", "Resetting to previous set of enabled states")
     733          if len(self.enabled)>0:
     734              self.enabled = self.__storedTemporarily.pop()
     735          else:
     736              raise ValueError("Trying to restore not-stored list of enabled " \
     737                               "states")
     738
     739
     740 -    def _getEnabled(self, nondefault=True, invert=False):
     741          """Return list of enabled states
     742
     743          :Parameters:
     744            nondefault : bool
     745              Either to return also states which are enabled simply by default
     746            invert : bool
     747              Would invert the meaning, ie would return disabled states
     748          """
     749          if invert:
     750              fmatch = lambda y: not self.isEnabled(y)
     751          else:
     752              fmatch = lambda y: self.isEnabled(y)
     753
     754          if nondefault:
     755              ffunc = fmatch
     756          else:
     757              ffunc = lambda y: fmatch(y) and \
     758                          self._items[y]._defaultenabled != self.isEnabled(y)
     759          return filter(ffunc, self.names)
     760
     761
     762 -    def _setEnabled(self, indexlist):
     763          """Given `indexlist` make only those in the list enabled
     764
     765          It might be handy to store set of enabled states and then to restore
     766          it later on. It can be easily accomplished now::
     767
     768          >>> from mvpa.misc.state import ClassWithCollections, StateVariable
     769          >>> class Blah(ClassWithCollections):
     770          ...   bleh = StateVariable(enabled=False, doc='Example')
     771          ...
     772          >>> blah = Blah()
     773          >>> states_enabled = blah.states.enabled
     774          >>> blah.states.enabled = ['bleh']
     775          >>> blah.states.enabled = states_enabled
     776          """
     777          for index in self._items.keys():
     778              self.enable(index, index in indexlist)
     779
     780
     781      # Properties
     782      enabled = property(fget=_getEnabled, fset=_setEnabled)
     783
     784
     785  ##################################################################
     786  # Base classes (and metaclass) which use collections
     787  #
     788
     789
     790  #
     791  # Helper dictionaries for AttributesCollector
     792  #
     793  _known_collections = {
     794      # Quite a generic one but mostly in classifiers
     795      'StateVariable': ("states", StateCollection),
     796      # For classifiers only
     797      'Parameter': ("params", ParameterCollection),
     798      'KernelParameter': ("kernel_params", ParameterCollection),
     799      # For datasets
     800      # XXX custom collections needed?
     801      'SampleAttribute':  ("sa", SampleAttributesCollection),
     802      'FeatureAttribute': ("fa", SampleAttributesCollection),
     803      'DatasetAttribute': ("dsa", SampleAttributesCollection),
     804      }
     805
     806
     807  _col2class = dict(_known_collections.values())
     808  """Mapping from collection name into Collection class"""
     809
     810
     811  _COLLECTIONS_ORDER = ['sa', 'fa', 'dsa',
     812                        'params', 'kernel_params', 'states']
     813
     814
     815 -class AttributesCollector(type):
     816      """Intended to collect and compose StateCollection for any child
     817      class of this metaclass
     818      """
     819
     820
     821 -    def __init__(cls, name, bases, dict):
     822
     823          if __debug__:
     824              debug(
     825                  "COLR",
     826                  "AttributesCollector call for %s.%s, where bases=%s, dict=%s " \
     827                  % (cls, name, bases, dict))
     828
     829          super(AttributesCollector, cls).__init__(name, bases, dict)
     830
     831          collections = {}
     832          for name, value in dict.iteritems():
     833              if isinstance(value, CollectableAttribute):
     834                  baseclassname = value.__class__.__name__
     835                  col = _known_collections[baseclassname][0]
     836                  # XXX should we allow to throw exceptions here?
     837                  if not collections.has_key(col):
     838                      collections[col] = {}
     839                  collections[col][name] = value
     840                  # and assign name if not yet was set
     841                  if value.name is None:
     842                      value._setName(name)
     843                  # !!! We do not keep copy of this attribute static in the class.
     844                  #     Due to below traversal of base classes, we should be
     845                  #     able to construct proper collections even in derived classes
     846                  delattr(cls, name)
     847
     848          # XXX can we first collect parent's states and then populate with ours?
     849          # TODO
     850
     851          for base in bases:
     852              if hasattr(base, "__metaclass__") and \
     853                     base.__metaclass__ == AttributesCollector:
     854                  # TODO take care about overriding one from super class
     855                  # for state in base.states:
     856                  #    if state[0] =
     857                  newcollections = base._collections_template
     858                  if len(newcollections) == 0:
     859                      continue
     860                  if __debug__:
     861                      debug("COLR",
     862                            "Collect collections %s for %s from %s" %
     863                            (newcollections, cls, base))
     864                  for col, collection in newcollections.iteritems():
     865                      newitems = collection.items
     866                      if collections.has_key(col):
     867                          collections[col].update(newitems)
     868                      else:
     869                          collections[col] = newitems
     870
     871
     872          if __debug__:
     873              debug("COLR",
     874                    "Creating StateCollection template %s with collections %s"
     875                    % (cls, collections.keys()))
     876
     877          # if there is an explicit
     878          if hasattr(cls, "_ATTRIBUTE_COLLECTIONS"):
     879              for col in cls._ATTRIBUTE_COLLECTIONS:
     880                  if not col in _col2class:
     881                      raise ValueError, \
     882                            "Requested collection %s is unknown to collector" % \
     883                            col
     884                  if not col in collections:
     885                      collections[col] = None
     886
     887          # TODO: check on conflict in names of Collections' items!  since
     888          # otherwise even order is not definite since we use dict for
     889          # collections.
     890          # XXX should we switch to tuple?
     891
     892          for col, colitems in collections.iteritems():
     893              collections[col] = _col2class[col](colitems)
     894
     895          setattr(cls, "_collections_template", collections)
     896
     897          #
     898          # Expand documentation for the class based on the listed
     899          # parameters an if it is stateful
     900          #
     901          # TODO -- figure nice way on how to alter __init__ doc directly...
     902          textwrapper = TextWrapper(subsequent_indent="    ",
     903                                    initial_indent="    ",
     904                                    width=70)
     905
     906          # Parameters
     907          paramsdoc = ""
     908          paramscols = []
     909          for col in ('params', 'kernel_params'):
     910              if collections.has_key(col):
     911                  paramscols.append(col)
     912                  # lets at least sort the parameters for consistent output
     913                  col_items = collections[col].items
     914                  params = [(v._instance_index, k) for k,v in col_items.iteritems()]
     915                  params.sort()
     916                  paramsdoc += '\n'.join(
     917                      [col_items[param].doc(indent='  ')
     918                       for index,param in params]) + '\n'
     919
     920          # Parameters collection could be taked hash of to decide if
     921          # any were changed? XXX may be not needed at all?
     922          setattr(cls, "_paramscols", paramscols)
     923
     924          # States doc
     925          statesdoc = ""
     926          if collections.has_key('states'):
     927              paramsdoc += """  enable_states : None or list of basestring
     928      Names of the state variables which should be enabled additionally
     929      to default ones
     930    disable_states : None or list of basestring
     931      Names of the state variables which should be disabled
     932  """
     933              statesdoc = "  * "
     934              statesdoc += '\n  * '.join(collections['states'].listing)
     935              statesdoc += "\n\n(States enabled by default are listed with `+`)"
     936              if __debug__:
     937                  debug("COLR", "Assigning __statesdoc to be %s" % statesdoc)
     938              setattr(cls, "_statesdoc", statesdoc)
     939
     940          if paramsdoc != "":
     941              if __debug__ and 'COLR' in debug.active:
     942                  debug("COLR", "Assigning __paramsdoc to be %s" % paramsdoc)
     943              setattr(cls, "_paramsdoc", paramsdoc)
     944
     945          if paramsdoc + statesdoc != "":
     946              cls.__doc__ = enhancedDocString(cls, *bases)
     947
     948
     949
     950 -class ClassWithCollections(object):
     951      """Base class for objects which contain any known collection
     952
     953      Classes inherited from this class gain ability to access
     954      collections and their items as simple attributes. Access to
     955      collection items "internals" is done via <collection_name> attribute
     956      and interface of a corresponding `Collection`.
     957      """
     958
     959      _DEV__doc__ = """
     960      TODO: rename 'descr'? -- it should simply
     961            be 'doc' -- no need to drag classes docstring imho.
     962      """
     963
     964      __metaclass__ = AttributesCollector
     965
     966 -    def __new__(cls, *args, **kwargs):
     967          """Initialize ClassWithCollections object
     968
     969          :Parameters:
     970            descr : basestring
     971              Description of the instance
     972          """
     973          self = super(ClassWithCollections, cls).__new__(cls)
     974
     975          s__dict__ = self.__dict__
     976
     977          # init variable
     978          # XXX: Added as pylint complained (rightfully) -- not sure if false
     979          # is the proper default
     980          self.__params_set = False
     981
     982          # need to check to avoid override of enabled states in the case
     983             # of multiple inheritance, like both ClassWithCollectionsl and Harvestable
     984          if not s__dict__.has_key('_collections'):
     985              s__class__ = self.__class__
     986
     987              collections = copy.deepcopy(s__class__._collections_template)
     988              s__dict__['_collections'] = collections
     989              s__dict__['_known_attribs'] = {}
     990              """Dictionary to contain 'links' to the collections from each
     991              known attribute. Is used to gain some speed up in lookup within
     992              __getattribute__ and __setattr__
     993              """
     994
     995              # Assign owner to all collections
     996              for col, collection in collections.iteritems():
     997                  if col in s__dict__:
     998                      raise ValueError, \
     999                            "Object %s has already attribute %s" % \
    1000                            (self, col)
    1001                  s__dict__[col] = collection
    1002                  collection.name = col
    1003                  collection.owner = self
    1004
    1005              self.__params_set = False
    1006
    1007          if __debug__:
    1008              descr = kwargs.get('descr', None)
    1009              debug("COL", "ClassWithCollections.__new__ was done "
    1010                    "for %s#%s with descr=%s" \
    1011                    % (s__class__.__name__, id(self), descr))
    1012
    1013          return self
    1014
    1015
    1016 -    def __init__(self, descr=None, **kwargs):
    1017
    1018          if not self.__params_set:
    1019              self.__descr = descr
    1020              """Set humane description for the object"""
    1021
    1022              # To avoid double initialization in case of multiple inheritance
    1023              self.__params_set = True
    1024
    1025              collections = self._collections
    1026              # Assign attributes values if they are given among
    1027              # **kwargs
    1028              for arg, argument in kwargs.items():
    1029                  set = False
    1030                  for collection in collections.itervalues():
    1031                      if collection._is_initializable(arg):
    1032                          collection._initialize(arg, argument)
    1033                          set = True
    1034                          break
    1035                  if set:
    1036                      trash = kwargs.pop(arg)
    1037                  else:
    1038                      known_params = reduce(
    1039                         lambda x,y:x+y,
    1040                         [x.items.keys() for x in collections.itervalues()], [])
    1041                      raise TypeError, \
    1042                            "Unexpected keyword argument %s=%s for %s." \
    1043                             % (arg, argument, self) \
    1044                            + " Valid parameters are %s" % known_params
    1045
    1046              ## Initialize other base classes
    1047              ##  commented out since it seems to be of no use for now
    1048              #if init_classes is not None:
    1049              #    # return back stateful arguments since they might be
    1050              #    # processed by underlying classes
    1051              #    kwargs.update(kwargs_stateful)
    1052              #    for cls in init_classes:
    1053              #        cls.__init__(self, **kwargs)
    1054              #else:
    1055              #    if len(kwargs)>0:
    1056              #        known_params = reduce(lambda x, y: x + y, \
    1057              #                            [x.items.keys() for x in collections],
    1058              #                            [])
    1059              #        raise TypeError, \
    1060              #              "Unknown parameters %s for %s." % (kwargs.keys(),
    1061              #                                                 self) \
    1062              #              + " Valid parameters are %s" % known_params
    1063          if __debug__:
    1064              debug("COL", "ClassWithCollections.__init__ was done "
    1065                    "for %s#%s with descr=%s" \
    1066                    % (self.__class__.__name__, id(self), descr))
    1067
    1068
    1069      #__doc__ = enhancedDocString('ClassWithCollections', locals())
    1070
    1071
    1072 -    def __getattribute__(self, index):
    1073          # return all private ones first since smth like __dict__ might be
    1074          # queried by copy before instance is __init__ed
    1075          if index[0] == '_':
    1076              return _object_getattribute(self, index)
    1077
    1078          s_dict = _object_getattribute(self, '__dict__')
    1079          # check if it is a known collection
    1080          collections = s_dict['_collections']
    1081          if index in collections:
    1082              return collections[index]
    1083
    1084          # check if it is a part of any collection
    1085          known_attribs = s_dict['_known_attribs']
    1086          if index in known_attribs:
    1087              return collections[known_attribs[index]]._items[index].value
    1088
    1089          # just a generic return
    1090          return _object_getattribute(self, index)
    1091
    1092
    1093 -    def __setattr__(self, index, value):
    1094          if index[0] == '_':
    1095              return _object_setattr(self, index, value)
    1096
    1097          # Check if a part of a collection, and set appropriately
    1098          s_dict = _object_getattribute(self, '__dict__')
    1099          known_attribs = s_dict['_known_attribs']
    1100          if index in known_attribs:
    1101              collections = s_dict['_collections']
    1102              collections[known_attribs[index]][index].value = value
    1103              return value
    1104
    1105          # Generic setattr
    1106          return _object_setattr(self, index, value)
    1107
    1108
    1109      # XXX not sure if we shouldn't implement anything else...
    1110 -    def reset(self):
    1111          for collection in self._collections.values():
    1112              collection.reset()
    1113
    1114
    1115 -    def __str__(self):
    1116          s = "%s:" % (self.__class__.__name__)
    1117          if self.__descr is not None:
    1118              s += "/%s " % self.__descr
    1119          if hasattr(self, "_collections"):
    1120              for col, collection in self._collections.iteritems():
    1121                  s += " %d %s:%s" % (len(collection.items), col, str(collection))
    1122          return s
    1123
    1124
    1125 -    def __repr__(self, prefixes=None, fullname=False):
    1126          """String definition of the object of ClassWithCollections object
    1127
    1128          :Parameters:
    1129            fullname : bool
    1130              Either to include full name of the module
    1131            prefixes : list of strings
    1132              What other prefixes to prepend to list of arguments
    1133          """
    1134          if prefixes is None:
    1135              prefixes = []
    1136          prefixes = prefixes[:]          # copy list
    1137          id_str = ""
    1138          module_str = ""
    1139          if __debug__:
    1140              if 'MODULE_IN_REPR' in debug.active:
    1141                  fullname = True
    1142              if 'ID_IN_REPR' in debug.active:
    1143                  id_str = '#%s' % id(self)
    1144
    1145          if fullname:
    1146              modulename = '%s' % self.__class__.__module__
    1147              if modulename != "__main__":
    1148                  module_str = "%s." % modulename
    1149
    1150          # Collections' attributes
    1151          collections = self._collections
    1152          # we want them in this particular order
    1153          for col in _COLLECTIONS_ORDER:
    1154              collection = collections.get(col, None)
    1155              if collection is None:
    1156                  continue
    1157              prefixes += collection._cls_repr()
    1158
    1159          # Description if present
    1160          descr = self.__descr
    1161          if descr is not None:
    1162              prefixes.append("descr=%s" % repr(descr))
    1163
    1164          return "%s%s(%s)%s" % (module_str, self.__class__.__name__,
    1165                                 ', '.join(prefixes), id_str)
    1166
    1167
    1168      descr = property(lambda self: self.__descr,
    1169                       doc="Description of the object if any")
    1170
    1171
    1172
    1173 -class Harvestable(ClassWithCollections):
    1174      """Classes inherited from this class intend to collect attributes
    1175      within internal processing.
    1176
    1177      Subclassing Harvestable we gain ability to collect any internal
    1178      data from the processing which is especially important if an
    1179      object performs something in loop and discards some intermidiate
    1180      possibly interesting results (like in case of
    1181      CrossValidatedTransferError and states of the trained classifier
    1182      or TransferError).
    1183
    1184      """
    1185
    1186      harvested = StateVariable(enabled=False, doc=
    1187         """Store specified attributes of classifiers at each split""")
    1188
    1189      _KNOWN_COPY_METHODS = [ None, 'copy', 'deepcopy' ]
    1190
    1191
    1192 -    def __init__(self, harvest_attribs=None, copy_attribs='copy', **kwargs):
    1193          """Initialize state of harvestable
    1194
    1195          :Parameters:
    1196            harvest_attribs : list of basestr or dicts
    1197              What attributes of call to store and return within
    1198              harvested state variable. If an item is a dictionary,
    1199              following keys are used ['name', 'copy']
    1200            copy_attribs : None or basestr
    1201              Default copying. If None -- no copying, 'copy'
    1202              - shallow copying, 'deepcopy' -- deepcopying
    1203
    1204          """
    1205          ClassWithCollections.__init__(self, **kwargs)
    1206
    1207          self.__atribs = harvest_attribs
    1208          self.__copy_attribs = copy_attribs
    1209
    1210          self._setAttribs(harvest_attribs)
    1211
    1212
    1213 -    def _setAttribs(self, attribs):
    1214          """Set attributes to harvest
    1215
    1216          Each attribute in self.__attribs must have following fields
    1217           - name : functional (or arbitrary if 'obj' or 'attr' is set)
    1218                    description of the thing to harvest,
    1219                    e.g. 'transerror.clf.training_time'
    1220           - obj : name of the object to harvest from (if empty,
    1221                   'self' is assumed),
    1222                   e.g 'transerror'
    1223           - attr : attribute of 'obj' to harvest,
    1224                   e.g. 'clf.training_time'
    1225           - copy : None, 'copy' or 'deepcopy' - way to copy attribute
    1226          """
    1227          if attribs:
    1228              # force the state
    1229              self.states.enable('harvested')
    1230              self.__attribs = []
    1231              for i, attrib in enumerate(attribs):
    1232                  if isinstance(attrib, dict):
    1233                      if not 'name' in attrib:
    1234                          raise ValueError, \
    1235                                "Harvestable: attribute must be a string or " + \
    1236                                "a dictionary with 'name'"
    1237                  else:
    1238                      attrib = {'name': attrib}
    1239
    1240                  # assign default method to copy
    1241                  if not 'copy' in attrib:
    1242                      attrib['copy'] = self.__copy_attribs
    1243
    1244                  # check copy method
    1245                  if not attrib['copy'] in self._KNOWN_COPY_METHODS:
    1246                      raise ValueError, "Unknown method %s. Known are %s" % \
    1247                            (attrib['copy'], self._KNOWN_COPY_METHODS)
    1248
    1249                  if not ('obj' in attrib or 'attr' in attrib):
    1250                      # Process the item to harvest
    1251                      # split into obj, attr. If obj is empty, then assume self
    1252                      split = attrib['name'].split('.', 1)
    1253                      if len(split)==1:
    1254                          obj, attr = split[0], None
    1255                      else:
    1256                          obj, attr = split
    1257                      attrib.update({'obj':obj, 'attr':attr})
    1258
    1259                  if attrib['obj'] == '':
    1260                      attrib['obj'] = 'self'
    1261
    1262                  # TODO: may be enabling of the states??
    1263
    1264                  self.__attribs.append(attrib)     # place value back
    1265          else:
    1266              # just to make sure it is not None or 0
    1267              self.__attribs = []
    1268
    1269
    1270 -    def _harvest(self, vars):
    1271          """The harvesting function: must obtain dictionary of variables
    1272          from the caller.
    1273
    1274          :Parameters:
    1275              vars : dict
    1276                  Dictionary of available data. Most often locals() could be
    1277                  passed as `vars`. Mention that desired to be harvested
    1278                  private attributes better be bound locally to some variable
    1279
    1280          :Returns:
    1281              nothing
    1282          """
    1283
    1284          if not self.states.isEnabled('harvested') or len(self.__attribs)==0:
    1285              return
    1286
    1287          if not self.states.isSet('harvested'):
    1288              self.harvested = dict([(a['name'], []) for a in self.__attribs])
    1289
    1290          for attrib in self.__attribs:
    1291              attrv = vars[attrib['obj']]
    1292
    1293              # access particular attribute if needed
    1294              if not attrib['attr'] is None:
    1295                  attrv = eval('attrv.%s' % attrib['attr'])
    1296
    1297              # copy the value if needed
    1298              attrv = {'copy':copy.copy,
    1299                       'deepcopy':copy.deepcopy,
    1300                       None:lambda x:x}[attrib['copy']](attrv)
    1301
    1302              self.harvested[attrib['name']].append(attrv)
    1303
    1304
    1305      harvest_attribs = property(fget=lambda self:self.__attribs,
    1306                                 fset=_setAttribs)
    1307