Package mvpa :: Package base :: Module externals

[hide private]
[frames] | no frames]

# Source Code for Module mvpa.base.externals

      1  # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
      2  # vi: set ft=python sts=4 ts=4 sw=4 et:
      3  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
      4  #
      5  #   See COPYING file distributed along with the PyMVPA package for the
      6  #   copyright and license terms.
      7  #
      8  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
      9  """Helper to verify presence of external libraries and modules
     10  """
     11
     12  __docformat__ = 'restructuredtext'
     13  import os
     14
     15  from mvpa.base import warning
     16  from mvpa import cfg
     17  from mvpa.misc.support import SmartVersion
     18
     19  if __debug__:
     20      from mvpa.base import debug
     21
     22 -class _VersionsChecker(dict):
     23      """Helper class to check the versions of the available externals
     24      """
     25 -    def __getitem__(self, key):
     26          if not self.has_key(key):
     27              exists(key, force=True, raiseException=True)
     28          return super(_VersionsChecker, self).__getitem__(key)
     29
     30  versions = _VersionsChecker()
     31  """Versions of available externals, as tuples
     32  """
     33
     34
     35 -def __check_scipy():
     36      """Check if scipy is present an if it is -- store its version
     37      """
     38      import warnings
     39      exists('numpy', raiseException='always')
     40      # To don't allow any crappy warning to sneak in
     41      warnings.simplefilter('ignore', DeprecationWarning)
     42      try:
     43          import scipy as sp
     44      except:
     45          warnings.simplefilter('default', DeprecationWarning)
     46          raise
     47      warnings.simplefilter('default', DeprecationWarning)
     48      # Infiltrate warnings if necessary
     49      numpy_ver = versions['numpy']
     50      scipy_ver = versions['scipy'] = SmartVersion(sp.__version__)
     51      # There is way too much deprecation warnings spit out onto the
     52      # user. Lets assume that they should be fixed by scipy 0.7.0 time
     53      if scipy_ver >= "0.6.0" and scipy_ver < "0.7.0" \
     54          and numpy_ver > "1.1.0":
     55          import warnings
     56          if not __debug__ or (__debug__ and not 'PY' in debug.active):
     57              if __debug__:
     58                  debug('EXT', "Setting up filters for numpy DeprecationWarnings")
     59              filter_lines = [
     60                  ('NumpyTest will be removed in the next release.*',
     61                   DeprecationWarning),
     62                  ('PyArray_FromDims: use PyArray_SimpleNew.',
     63                   DeprecationWarning),
     64                  ('PyArray_FromDimsAndDataAndDescr: use PyArray_NewFromDescr.',
     65                   DeprecationWarning),
     66                  # Trick re.match, since in warnings absent re.DOTALL in re.compile
     67                  ('[\na-z \t0-9]*The original semantics of histogram is scheduled to be.*'
     68                   '[\na-z \t0-9]*', Warning) ]
     69              for f, w in filter_lines:
     70                  warnings.filterwarnings('ignore', f, w)
     71
     72
     73 -def __check_numpy():
     74      """Check if numpy is present (it must be) an if it is -- store its version
     75      """
     76      import numpy as N
     77      versions['numpy'] = SmartVersion(N.__version__)
     78
     79
     80 -def __check_pywt(features=None):
     81      """Check for available functionality within pywt
     82
     83      :Parameters:
     84        features : list of basestring
     85          List of known features to check such as 'wp reconstruct',
     86          'wp reconstruct fixed'
     87      """
     88      import pywt
     89      import numpy as N
     90      data = N.array([ 0.57316901,  0.65292526,  0.75266733,  0.67020084,  0.46505364,
     91                       0.76478331,  0.33034164,  0.49165547,  0.32979941,  0.09696717,
     92                       0.72552711,  0.4138999 ,  0.54460628,  0.786351  ,  0.50096306,
     93                       0.72436454, 0.2193098 , -0.0135051 ,  0.34283984,  0.65596245,
     94                       0.49598417,  0.39935064,  0.26370727,  0.05572373,  0.40194438,
     95                       0.47004551,  0.60327258,  0.25628266,  0.32964893,  0.24009889,])
     96      mode = 'per'
     97      wp = pywt.WaveletPacket(data, 'sym2', mode)
     98      wp2 = pywt.WaveletPacket(data=None, wavelet='sym2', mode=mode)
     99      try:
    100          for node in wp.get_level(2): wp2[node.path] = node.data
    101      except:
    102          raise ImportError, \
    103                 "Failed to reconstruct WP by specifying data in the layer"
    104
    105      if 'wp reconstruct fixed' in features:
    106          rec = wp2.reconstruct()
    107          if N.linalg.norm(rec[:len(data)] - data) > 1e-3:
    108              raise ImportError, \
    109                    "Failed to reconstruct WP correctly"
    110      return True
    111
    112
    113 -def __check_libsvm_verbosity_control():
    114      """Check for available verbose control functionality
    115      """
    116      import mvpa.clfs.libsvmc._svmc as _svmc
    117      try:
    118          _svmc.svm_set_verbosity(0)
    119      except:
    120          raise ImportError, "Provided version of libsvm has no way to control " \
    121                "its level of verbosity"
    122
    123 -def __check_shogun(bottom_version, custom_versions=[]):
    124      """Check if version of shogun is high enough (or custom known) to
    125      be enabled in the testsuite
    126
    127      :Parameters:
    128        bottom_version : int
    129          Bottom version which must be satisfied
    130        custom_versions : list of int
    131          Arbitrary list of versions which could got patched for
    132          a specific issue
    133      """
    134      import shogun.Classifier as __sc
    135      ver = __sc.Version_get_version_revision()
    136      if (ver in custom_versions) or (ver >= bottom_version):
    137          return True
    138      else:
    139          raise ImportError, 'Version %s is smaller than needed %s' % \
    140                (ver, bottom_version)
    141
    142
    143 -def __check_weave():
    144      """Apparently presence of scipy is not sufficient since some
    145      versions experience problems. E.g. in Sep,Oct 2008 lenny's weave
    146      failed to work. May be some other converter could work (? See
    147      http://lists.debian.org/debian-devel/2008/08/msg00730.html for a
    148      similar report.
    149
    150      Following simple snippet checks compilation of the basic code using
    151      weave
    152      """
    153      from scipy import weave
    154      from scipy.weave import converters, build_tools
    155      import numpy as N
    156      # to shut weave up
    157      import sys
    158      # we can't rely on weave at all at the restoring argv. On etch box
    159      # restore_sys_argv() is apparently is insufficient
    160      oargv = sys.argv[:]
    161      ostdout = sys.stdout
    162      if not( __debug__ and 'EXT_' in debug.active):
    163          from StringIO import StringIO
    164          sys.stdout = StringIO()
    165          # *nix specific solution to shut weave up.
    166          # Some users must complain and someone
    167          # needs to fix this to become more generic.
    168          cargs = [">/dev/null", "2>&1"]
    169      else:
    170          cargs = []
    171      fmsg = None
    172      try:
    173          data = N.array([1,2,3])
    174          counter = weave.inline("data[0]=fabs(-1);", ['data'],
    175                                 type_converters=converters.blitz,
    176                                 verbose=0,
    177                                 extra_compile_args=cargs,
    178                                 compiler = 'gcc')
    179      except Exception, e:
    180          fmsg = "Failed to build simple weave sample." \
    181                 " Exception was %s" % str(e)
    182
    183      sys.stdout = ostdout
    184      # needed to fix sweave which might "forget" to restore sysv
    185      # build_tools.restore_sys_argv()
    186      sys.argv = oargv
    187      if fmsg is not None:
    188          raise ImportError, fmsg
    189      else:
    190          return "Everything is cool"
    191
    192
    193 -def __check_atlas_family(family):
    194      # XXX I guess pylint will dislike it a lot
    195      from mvpa.atlases.warehouse import KNOWN_ATLAS_FAMILIES
    196      names, pathpattern = KNOWN_ATLAS_FAMILIES[family]
    197      filename = pathpattern % {'name':names[0]}
    198      if not os.path.exists(filename):
    199          raise ImportError, "Cannot find file %s for atlas family %s" \
    200                % (filename, family)
    201      pass
    202
    203
    204 -def __check_stablerdist():
    205      import scipy.stats
    206      import numpy as N
    207      ## Unfortunately 0.7.0 hasn't fixed the issue so no chance but to do
    208      ## a proper numerical test here
    209      try:
    210          scipy.stats.rdist(1.32, 0, 1).cdf(-1.0 + N.finfo(float).eps)
    211          # Actually previous test is insufficient for 0.6, so enabling
    212          # elderly test on top
    213          # ATM all known implementations which implement custom cdf for
    214          #     rdist are misbehaving, so there should be no _cdf
    215          if '_cdf' in scipy.stats.distributions.rdist_gen.__dict__.keys():
    216              raise ImportError, \
    217                    "scipy.stats carries misbehaving rdist distribution"
    218      except ZeroDivisionError:
    219          raise RuntimeError, "RDist in scipy is still unstable on the boundaries"
    220
    221
    222 -def __check_rv_discrete_ppf():
    223      """Unfortunately 0.6.0-12 of scipy pukes on simple ppf
    224      """
    225      import scipy.stats
    226      try:
    227          bdist = scipy.stats.binom(100, 0.5)
    228          bdist.ppf(0.9)
    229      except TypeError:
    230          raise RuntimeError, "pmf is broken in discrete dists of scipy.stats"
    231
    232
    233 -def __check_in_ipython():
    234      # figure out if ran within IPython
    235      if '__IPYTHON__' in globals()['__builtins__']:
    236          return
    237      raise RuntimeError, "Not running in IPython session"
    238
    239 -def __check_openopt():
    240      try:
    241          import openopt as _
    242          return
    243      except ImportError:
    244          pass
    245      import scikits.openopt as _
    246      return
    247
    248 -def __check_matplotlib():
    249      """Check for presence of matplotlib and set backend if requested."""
    250      import matplotlib
    251      backend = cfg.get('matplotlib', 'backend')
    252      if backend:
    253          matplotlib.use(backend)
    254
    255 -def __check_pylab():
    256      """Check if matplotlib is there and then pylab"""
    257      exists('matplotlib', raiseException='always')
    258      import pylab as P
    259
    260 -def __check_pylab_plottable():
    261      """Simple check either we can plot anything using pylab.
    262
    263      Primary use in unittests
    264      """
    265      try:
    266          exists('pylab', raiseException='always')
    267          import pylab as P
    268          fig = P.figure()
    269          P.plot([1,2], [1,2])
    270          P.close(fig)
    271      except:
    272          raise RuntimeError, "Cannot plot in pylab"
    273      return True
    274
    275
    276 -def __check_griddata():
    277      """griddata might be independent module or part of mlab
    278      """
    279
    280      try:
    281          from griddata import griddata as __
    282          return True
    283      except ImportError:
    284          if __debug__:
    285              debug('EXT_', 'No python-griddata available')
    286
    287      from matplotlib.mlab import griddata as __
    288      return True
    289
    290
    291 -def __check_reportlab():
    292      import reportlab as rl
    293      versions['reportlab'] = SmartVersion(rl.Version)
    294
    295
    296 -def __check_rpy():
    297      """Check either rpy is available and also set it for the sane execution
    298      """
    299      #import rpy_options
    300      #rpy_options.set_options(VERBOSE=False, SETUP_READ_CONSOLE=False) # SETUP_WRITE_CONSOLE=False)
    301      #rpy_options.set_options(VERBOSE=False, SETUP_WRITE_CONSOLE=False) # SETUP_WRITE_CONSOLE=False)
    302      #    if not cfg.get('rpy', 'read_console', default=False):
    303      #        print "no read"
    304      #        rpy_options.set_options(SETUP_READ_CONSOLE=False)
    305      #    if not cfg.get('rpy', 'write_console', default=False):
    306      #        print "no write"
    307      #        rpy_options.set_options(SETUP_WRITE_CONSOLE=False)
    308      import rpy
    309      if not cfg.getboolean('rpy', 'interactive', default=True) \
    310             and (rpy.get_rpy_input() is rpy.rpy_io.rpy_input):
    311          if __debug__:
    312              debug('EXT_', "RPy: providing dummy callback for input to return '1'")
    313          def input1(*args): return "1"      # which is "1: abort (with core dump, if enabled)"
    314          rpy.set_rpy_input(input1)
    315
    316
    317  # contains list of available (optional) external classifier extensions
    318  _KNOWN = {'libsvm':'import mvpa.clfs.libsvmc._svm as __; x=__.convert2SVMNode',
    319            'libsvm verbosity control':'__check_libsvm_verbosity_control();',
    320            'nifti':'from nifti import NiftiImage as __',
    321            'nifti ge 0.20090205.1':
    322                  'from nifti.clib import detachDataFromImage as __',
    323            'ctypes':'import ctypes as __',
    324            'shogun':'import shogun as __',
    325            'shogun.krr': 'import shogun.Regression as __; x=__.KRR',
    326            'shogun.mpd': 'import shogun.Classifier as __; x=__.MPDSVM',
    327            'shogun.lightsvm': 'import shogun.Classifier as __; x=__.SVMLight',
    328            'shogun.svrlight': 'from shogun.Regression import SVRLight as __',
    329            'numpy': "__check_numpy()",
    330            'scipy': "__check_scipy()",
    331            'good scipy.stats.rdist': "__check_stablerdist()",
    332            'good scipy.stats.rv_discrete.ppf': "__check_rv_discrete_ppf()",
    333            'weave': "__check_weave()",
    334            'pywt': "import pywt as __",
    335            'pywt wp reconstruct': "__check_pywt(['wp reconstruct'])",
    336            'pywt wp reconstruct fixed': "__check_pywt(['wp reconstruct fixed'])",
    337            'rpy': "__check_rpy()",
    338            'lars': "exists('rpy', raiseException='always'); import rpy; rpy.r.library('lars')",
    339            'elasticnet': "exists('rpy', raiseException='always'); import rpy; rpy.r.library('elasticnet')",
    340            'glmnet': "exists('rpy', raiseException='always'); import rpy; rpy.r.library('glmnet')",
    341            'matplotlib': "__check_matplotlib()",
    342            'pylab': "__check_pylab()",
    343            'pylab plottable': "__check_pylab_plottable()",
    344            'openopt': "__check_openopt()",
    345            'mdp': "import mdp as __",
    346            'mdp ge 2.4': "from mdp.nodes import LLENode as __",
    347            'sg_fixedcachesize': "__check_shogun(3043, [2456])",
    348             # 3318 corresponds to release 0.6.4
    349            'sg ge 0.6.4': "__check_shogun(3318)",
    350            'hcluster': "import hcluster as __",
    351            'griddata': "__check_griddata()",
    352            'cPickle': "import cPickle as __",
    353            'gzip': "import gzip as __",
    354            'lxml': "from lxml import objectify as __",
    355            'atlas_pymvpa': "__check_atlas_family('pymvpa')",
    356            'atlas_fsl': "__check_atlas_family('fsl')",
    357            'running ipython env': "__check_in_ipython()",
    358            'reportlab': "__check_reportlab()",
    359            'nose': "import nose as __",
    360            }
    361
    362
    363 -def exists(dep, force=False, raiseException=False, issueWarning=None):
    364      """
    365      Test whether a known dependency is installed on the system.
    366
    367      This method allows us to test for individual dependencies without
    368      testing all known dependencies. It also ensures that we only test
    369      for a dependency once.
    370
    371      :Parameters:
    372        dep : string or list of string
    373          The dependency key(s) to test.
    374        force : boolean
    375          Whether to force the test even if it has already been
    376          performed.
    377        raiseException : boolean or 'always'
    378          Whether to raise RuntimeError if dependency is missing.
    379          If True, it is still conditioned on the global setting
    380          MVPA_EXTERNALS_RAISE_EXCEPTION, while would raise exception
    381          if missing despite the configuration if 'always'.
    382        issueWarning : string or None or True
    383          If string, warning with given message would be thrown.
    384          If True, standard message would be used for the warning
    385          text.
    386      """
    387      # if we are provided with a list of deps - go through all of them
    388      if isinstance(dep, list) or isinstance(dep, tuple):
    389          results = [ exists(dep_, force, raiseException) for dep_ in dep ]
    390          return bool(reduce(lambda x,y: x and y, results, True))
    391
    392      # where to look in cfg
    393      cfgid = 'have ' + dep
    394
    395      # pre-handle raiseException according to the global settings and local argument
    396      if isinstance(raiseException, str):
    397          if raiseException.lower() == 'always':
    398              raiseException = True
    399          else:
    400              raise ValueError("Unknown value of raiseException=%s. "
    401                               "Must be bool or 'always'" % raiseException)
    402      else: # must be bool conditioned on the global settings
    403          raiseException = raiseException \
    404                  and cfg.getboolean('externals', 'raise exception', True)
    405
    406      # prevent unnecessarry testing
    407      if cfg.has_option('externals', cfgid) \
    408         and not cfg.getboolean('externals', 'retest', default='no') \
    409         and not force:
    410          if __debug__:
    411              debug('EXT', "Skip retesting for '%s'." % dep)
    412
    413          # check whether an exception should be raised, even though the external
    414          # was already tested previously
    415          if not cfg.getboolean('externals', cfgid) and raiseException:
    416              raise RuntimeError, "Required external '%s' was not found" % dep
    417          return cfg.getboolean('externals', cfgid)
    418
    419
    420      # determine availability of external (non-cached)
    421
    422      # default to 'not found'
    423      result = False
    424
    425      if not _KNOWN.has_key(dep):
    426          raise ValueError, "%s is not a known dependency key." % (dep)
    427      else:
    428          # try and load the specific dependency
    429          if __debug__:
    430              debug('EXT', "Checking for the presence of %s" % dep)
    431
    432          # Exceptions which are silently caught while running tests for externals
    433          _caught_exceptions = [ImportError, AttributeError, RuntimeError]
    434
    435          # check whether RPy is involved and catch its excpetions as well.
    436          # however, try to determine whether this is really necessary, as
    437          # importing RPy also involved starting a full-blown R session, which can
    438          # take seconds and therefore is quite nasty...
    439          if dep.count('rpy') or _KNOWN[dep].count('rpy'):
    440              try:
    441                  if dep == 'rpy':
    442                      __check_rpy()          # needed to be run to adjust options first
    443                  else:
    444                      if exists('rpy'):
    445                          # otherwise no need to add anything -- test
    446                          # would fail since rpy isn't available
    447                          from rpy import RException
    448                          _caught_exceptions += [RException]
    449              except:
    450                  pass
    451
    452
    453          estr = ''
    454          try:
    455              exec _KNOWN[dep]
    456              result = True
    457          except tuple(_caught_exceptions), e:
    458              estr = ". Caught exception was: " + str(e)
    459
    460          if __debug__:
    461              debug('EXT', "Presence of %s is%s verified%s" %
    462                    (dep, {True:'', False:' NOT'}[result], estr))
    463
    464      if not result:
    465          if raiseException:
    466              raise RuntimeError, "Required external '%s' was not found" % dep
    467          if issueWarning is not None \
    468                 and cfg.getboolean('externals', 'issue warning', True):
    469              if issueWarning is True:
    470                  warning("Required external '%s' was not found" % dep)
    471              else:
    472                  warning(issueWarning)
    473
    474
    475      # store result in config manager
    476      if not cfg.has_section('externals'):
    477          cfg.add_section('externals')
    478      if result:
    479          cfg.set('externals', 'have ' + dep, 'yes')
    480      else:
    481          cfg.set('externals', 'have ' + dep, 'no')
    482
    483      return result
    484
    485
    486 -def testAllDependencies(force=False):
    487      """
    488      Test for all known dependencies.
    489
    490      :Parameters:
    491        force : boolean
    492          Whether to force the test even if it has already been
    493          performed.
    494
    495      """
    496      # loop over all known dependencies
    497      for dep in _KNOWN:
    498          if not exists(dep, force):
    499              warning("%s is not available." % dep)
    500
    501      if __debug__:
    502          debug('EXT', 'The following optional externals are present: %s' \
    503                       % [ k[5:] for k in cfg.options('externals')
    504                              if k.startswith('have') \
    505                              and cfg.getboolean('externals', k) == True ])
    506