Package mvpa :: Package base

[hide private]
[frames] | no frames]

# Source Code for Package mvpa.base

      1  # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
      2  # vi: set ft=python sts=4 ts=4 sw=4 et:
      3  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
      4  #
      5  #   See COPYING file distributed along with the PyMVPA package for the
      6  #   copyright and license terms.
      7  #
      8  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
      9  """Base functionality of PyMVPA
     10
     11  Module Organization
     12  ===================
     13
     14  mvpa.base module contains various modules which are used through out
     15  PyMVPA code, and are generic building blocks
     16
     17  .. packagetree::
     18     :style: UML
     19
     20  :group Basic: externals, config, verbosity, dochelpers
     21  """
     22
     23  __docformat__ = 'restructuredtext'
     24
     25
     26  import sys
     27  from mvpa.base.config import ConfigManager
     28  from mvpa.base.verbosity import LevelLogger, OnceLogger
     29
     30  #
     31  # Setup verbose and debug outputs
     32  #
     33 -class _SingletonType(type):
     34      """Simple singleton implementation adjusted from
     35      http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/412551
     36      """
     37 -    def __init__(mcs, *args):
     38          type.__init__(mcs, *args)
     39          mcs._instances = {}
     40
     41 -    def __call__(mcs, sid, instance, *args):
     42          if not sid in mcs._instances:
     43              mcs._instances[sid] = instance
     44          return mcs._instances[sid]
     45
     46 -class __Singleton:
     47      """To ensure single instance of a class instantiation (object)
     48
     49      """
     50
     51      __metaclass__ = _SingletonType
     52 -    def __init__(self, *args):
     53          pass
     54      # Provided __call__ just to make silly pylint happy
     55 -    def __call__(self):
     56          raise NotImplementedError
     57
     58  #
     59  # As the very first step: Setup configuration registry instance and
     60  # read all configuration settings from files and env variables
     61  #
     62  cfg = __Singleton('cfg', ConfigManager())
     63
     64  verbose = __Singleton("verbose", LevelLogger(
     65      handlers = cfg.get('verbose', 'output', default='stdout').split(',')))
     66
     67  # Not supported/explained/used by now since verbose(0, is to print errors
     68  #error = __Singleton("error", LevelLogger(
     69  #    handlers=environ.get('MVPA_ERROR_OUTPUT', 'stderr').split(',')))
     70
     71  # Levels for verbose
     72  # 0 -- nothing besides errors
     73  # 1 -- high level stuff -- top level operation or file operations
     74  # 2 -- cmdline handling
     75  # 3 --
     76  # 4 -- computation/algorithm relevant thingies
     77
     78  # Helper for errors printing
     79 -def error(msg, critical=True):
     80      """Helper function to output errors in a consistent way.
     81
     82      :Parameters:
     83        msg : string
     84          Actual error message (will be prefixed with ERROR:)
     85        critical : bool
     86          If critical error -- exit with
     87      """
     88      verbose(0, "ERROR: " + msg)
     89      if critical:
     90          raise sys.exit(1)
     91
     92  # Lets check if environment can tell us smth
     93  if cfg.has_option('general', 'verbose'):
     94      verbose.level = cfg.getint('general', 'verbose')
     95
     96
     97 -class WarningLog(OnceLogger):
     98      """Logging class of messsages to be printed just once per each message
     99
    100      """
    101
    102 -    def __init__(self, btlevels=10, btdefault=False,
    103                   maxcount=1, *args, **kwargs):
    104          """Define Warning logger.
    105
    106          It is defined by
    107            btlevels : int
    108              how many levels of backtrack to print to give a hint on WTF
    109            btdefault : bool
    110              if to print backtrace for all warnings at all
    111            maxcount : int
    112              how many times to print each warning
    113          """
    114          OnceLogger.__init__(self, *args, **kwargs)
    115          self.__btlevels = btlevels
    116          self.__btdefault = btdefault
    117          self.__maxcount = maxcount
    118          self.__explanation_seen = False
    119
    120
    121 -    def __call__(self, msg, bt=None):
    122          import traceback
    123          if bt is None:
    124              bt = self.__btdefault
    125          tb = traceback.extract_stack(limit=2)
    126          msgid = repr(tb[-2])         # take parent as the source of ID
    127          fullmsg = "WARNING: %s" % msg
    128          if not self.__explanation_seen:
    129              self.__explanation_seen = True
    130              fullmsg += "\n * Please note: warnings are "  + \
    131                    "printed only once, but underlying problem might " + \
    132                    "occur many times *"
    133          if bt and self.__btlevels > 0:
    134              fullmsg += "Top-most backtrace:\n"
    135              fullmsg += reduce(lambda x, y: x + "\t%s:%d in %s where '%s'\n" % \
    136                                y,
    137                                traceback.extract_stack(limit=self.__btlevels),
    138                                "")
    139
    140          OnceLogger.__call__(self, msgid, fullmsg, self.__maxcount)
    141
    142
    143 -    def _setMaxCount(self, value):
    144          """Set maxcount for the warning"""
    145          self.__maxcount = value
    146
    147      maxcount = property(fget=lambda x:x.__maxcount, fset=_setMaxCount)
    148
    149  # XXX what is 'bt'? Maybe more verbose name?
    150  if cfg.has_option('warnings', 'bt'):
    151      warnings_btlevels = cfg.getint('warnings', 'bt')
    152      warnings_bt = True
    153  else:
    154      warnings_btlevels = 10
    155      warnings_bt = False
    156
    157  if cfg.has_option('warnings', 'count'):
    158      warnings_maxcount = cfg.getint('warnings', 'count')
    159  else:
    160      warnings_maxcount = 1
    161
    162  warning = WarningLog(
    163      handlers={
    164          False: cfg.get('warnings', 'output', default='stdout').split(','),
    165          True: []}[cfg.getboolean('warnings', 'suppress', default=False)],
    166      btlevels=warnings_btlevels,
    167      btdefault=warnings_bt,
    168      maxcount=warnings_maxcount
    169      )
    170
    171
    172  if __debug__:
    173      from mvpa.base.verbosity import DebugLogger
    174      # NOTE: all calls to debug must be preconditioned with
    175      # if __debug__:
    176
    177      debug = __Singleton("debug", DebugLogger(
    178          handlers=cfg.get('debug', 'output', default='stdout').split(',')))
    179
    180      # set some debugging matricses to report
    181      # debug.registerMetric('vmem')
    182
    183      # List agreed sets for debug
    184      debug.register('PY',   "No suppression of various warnings (numpy, scipy) etc.")
    185      debug.register('DBG',  "Debug output itself")
    186      debug.register('DOCH', "Doc helpers")
    187      debug.register('INIT', "Just sequence of inits")
    188      debug.register('RANDOM', "Random number generation")
    189      debug.register('EXT',  "External dependencies")
    190      debug.register('EXT_', "External dependencies (verbose)")
    191      debug.register('TEST', "Debug unittests")
    192      debug.register('MODULE_IN_REPR', "Include module path in __repr__")
    193      debug.register('ID_IN_REPR', "Include id in __repr__")
    194      debug.register('CMDLINE', "Handling of command line parameters")
    195
    196      debug.register('DG',   "Data generators")
    197      debug.register('LAZY', "Miscelaneous 'lazy' evaluations")
    198      debug.register('LOOP', "Support's loop construct")
    199      debug.register('PLR',  "PLR call")
    200      debug.register('SLC',  "Searchlight call")
    201      debug.register('SA',   "Sensitivity analyzers")
    202      debug.register('SOM',  "Self-organizing-maps (SOM)")
    203      debug.register('IRELIEF', "Various I-RELIEFs")
    204      debug.register('SA_',  "Sensitivity analyzers (verbose)")
    205      debug.register('PSA',  "Perturbation analyzer call")
    206      debug.register('RFEC', "Recursive Feature Elimination call")
    207      debug.register('RFEC_', "Recursive Feature Elimination call (verbose)")
    208      debug.register('IFSC', "Incremental Feature Search call")
    209      debug.register('DS',   "*Dataset")
    210      debug.register('DS_NIFTI', "NiftiDataset(s)")
    211      debug.register('DS_',  "*Dataset (verbose)")
    212      debug.register('DS_ID',   "ID Datasets")
    213      debug.register('DS_STATS',"Datasets statistics")
    214      debug.register('SPL',   "*Splitter")
    215
    216      debug.register('TRAN',  "Transformers")
    217      debug.register('TRAN_', "Transformers (verbose)")
    218
    219      # CHECKs
    220      debug.register('CHECK_DS_SELECT',
    221                     "Check in dataset.select() for sorted and unique indexes")
    222      debug.register('CHECK_DS_SORTED', "Check in datasets for sorted")
    223      debug.register('CHECK_IDS_SORTED',
    224                     "Check for ids being sorted in mappers")
    225      debug.register('CHECK_TRAINED',
    226                     "Checking in checking if clf was trained on given dataset")
    227      debug.register('CHECK_RETRAIN', "Checking in retraining/retesting")
    228      debug.register('CHECK_STABILITY', "Checking for numerical stability")
    229      debug.register('ENFORCE_STATES_ENABLED', "Forcing all states to be enabled")
    230
    231      debug.register('MAP',   "*Mapper")
    232      debug.register('MAP_',  "*Mapper (verbose)")
    233
    234      debug.register('COL',  "Generic Collectable")
    235      debug.register('UATTR', "Attributes with unique")
    236      debug.register('ST',   "State")
    237      debug.register('STV',  "State Variable")
    238      debug.register('COLR', "Collector for states and classifier parameters")
    239      debug.register('ES',   "Element selectors")
    240
    241      debug.register('CLF',    "Base Classifiers")
    242      debug.register('CLF_',   "Base Classifiers (verbose)")
    243      #debug.register('CLF_TB',
    244      #    "Report traceback in train/predict. Helps to resolve WTF calls it")
    245      debug.register('CLFBST', "BoostClassifier")
    246      #debug.register('CLFBST_TB', "BoostClassifier traceback")
    247      debug.register('CLFBIN', "BinaryClassifier")
    248      debug.register('CLFTREE', "TreeClassifier")
    249      debug.register('CLFMC',  "MulticlassClassifier")
    250      debug.register('CLFSPL', "SplitClassifier")
    251      debug.register('CLFSPL_',"SplitClassifier (verbose)")
    252      debug.register('CLFFS',  "FeatureSelectionClassifier")
    253      debug.register('CLFFS_', "FeatureSelectionClassifier (verbose)")
    254
    255      debug.register('STAT',   "Statistics estimates")
    256      debug.register('STAT_',  "Statistics estimates (verbose)")
    257      debug.register('STAT__', "Statistics estimates (very verbose)")
    258      debug.register('STATMC', "Progress in Monte-Carlo estimation")
    259
    260      debug.register('FS',     "FeatureSelections")
    261      debug.register('FS_',    "FeatureSelections (verbose)")
    262      debug.register('FSPL',   "FeatureSelectionPipeline")
    263
    264      debug.register('SVM',    "SVM")
    265      debug.register('SVM_',   "SVM (verbose)")
    266      debug.register('LIBSVM', "Internal libsvm output")
    267
    268      debug.register('SMLR',    "SMLR")
    269      debug.register('SMLR_',   "SMLR verbose")
    270
    271      debug.register('LARS',    "LARS")
    272      debug.register('LARS_',   "LARS (verbose)")
    273
    274      debug.register('ENET',    "ENET")
    275      debug.register('ENET_',   "ENET (verbose)")
    276
    277      debug.register('GLMNET',    "GLMNET")
    278      debug.register('GLMNET_',   "GLMNET (verbose)")
    279
    280      debug.register('GNB',     "GNB - Gaussian Naive Bayes")
    281
    282      debug.register('GPR',     "GPR")
    283      debug.register('GPR_WEIGHTS', "Track progress of GPRWeights computation")
    284      debug.register('KERNEL',  "Kernels module")
    285      debug.register('MOD_SEL', "Model Selector (also makes openopt's iprint=0)")
    286      debug.register('OPENOPT', "OpenOpt toolbox verbose (iprint=1)")
    287
    288      debug.register('SG',  "PyMVPA SG wrapping")
    289      debug.register('SG_', "PyMVPA SG wrapping verbose")
    290      debug.register('SG__', "PyMVPA SG wrapping debug")
    291      debug.register('SG_SVM', "Internal shogun debug output for SVM itself")
    292      debug.register('SG_FEATURES', "Internal shogun debug output for features")
    293      debug.register('SG_LABELS', "Internal shogun debug output for labels")
    294      debug.register('SG_KERNELS', "Internal shogun debug output for kernels")
    295      debug.register('SG_PROGRESS',
    296                     "Internal shogun progress bar during computation")
    297
    298      debug.register('IOH',    "IO Helpers")
    299      debug.register('IO_HAM', "Hamster")
    300      debug.register('CM',   "Confusion matrix computation")
    301      debug.register('ROC',  "ROC analysis")
    302      debug.register('CROSSC', "Cross-validation call")
    303      debug.register('CERR', "Various ClassifierErrors")
    304
    305      debug.register('ATL',    "Atlases")
    306      debug.register('ATL_',   "Atlases (verbose)")
    307      debug.register('ATL__',  "Atlases (very verbose)")
    308
    309      debug.register('REP',    "Reports")
    310      debug.register('REP_',   "Reports (verbose)")
    311
    312      # Lets check if environment can tell us smth
    313      if cfg.has_option('general', 'debug'):
    314          debug.setActiveFromString(cfg.get('general', 'debug'))
    315
    316      # Lets check if environment can tell us smth
    317      if cfg.has_option('debug', 'metrics'):
    318          debug.registerMetric(cfg.get('debug', 'metrics').split(","))
    319
    320
    321
    322  if __debug__:
    323      debug('INIT', 'mvpa.base end')
    324