Package logilab :: Package common :: Module modutils

[frames] | no frames]

# Source Code for Module logilab.common.modutils

      1  # -*- coding: utf-8 -*-
      2  # copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
      3  # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
      4  #
      5  # This file is part of logilab-common.
      6  #
      7  # logilab-common is free software: you can redistribute it and/or modify it under
      8  # the terms of the GNU Lesser General Public License as published by the Free
      9  # Software Foundation, either version 2.1 of the License, or (at your option) any
     10  # later version.
     11  #
     12  # logilab-common is distributed in the hope that it will be useful, but WITHOUT
     13  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
     14  # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
     15  # details.
     16  #
     17  # You should have received a copy of the GNU Lesser General Public License along
     18  # with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
     19  """Python modules manipulation utility functions.
     20
     21  :type PY_SOURCE_EXTS: tuple(str)
     22  :var PY_SOURCE_EXTS: list of possible python source file extension
     23
     24  :type STD_LIB_DIR: str
     25  :var STD_LIB_DIR: directory where standard modules are located
     26
     27  :type BUILTIN_MODULES: dict
     28  :var BUILTIN_MODULES: dictionary with builtin module names as key
     29  """
     30
     31  __docformat__ = "restructuredtext en"
     32
     33  import sys
     34  import os
     35  from os.path import splitext, join, abspath, isdir, dirname, exists, basename
     36  from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY
     37  from distutils.sysconfig import get_config_var, get_python_lib, get_python_version
     38  from distutils.errors import DistutilsPlatformError
     39
     40  from six.moves import range
     41
     42  try:
     43      import zipimport
     44  except ImportError:
     45      zipimport = None
     46
     47  ZIPFILE = object()
     48
     49  from logilab.common import STD_BLACKLIST, _handle_blacklist
     50
     51  # Notes about STD_LIB_DIR
     52  # Consider arch-specific installation for STD_LIB_DIR definition
     53  # :mod:`distutils.sysconfig` contains to much hardcoded values to rely on
     54  #
     55  # :see: `Problems with /usr/lib64 builds <http://bugs.python.org/issue1294959>`_
     56  # :see: `FHS <http://www.pathname.com/fhs/pub/fhs-2.3.html#LIBLTQUALGTALTERNATEFORMATESSENTIAL>`_
     57  if sys.platform.startswith('win'):
     58      PY_SOURCE_EXTS = ('py', 'pyw')
     59      PY_COMPILED_EXTS = ('dll', 'pyd')
     60  else:
     61      PY_SOURCE_EXTS = ('py',)
     62      PY_COMPILED_EXTS = ('so',)
     63
     64  try:
     65      STD_LIB_DIR = get_python_lib(standard_lib=True)
     66  # get_python_lib(standard_lib=1) is not available on pypy, set STD_LIB_DIR to
     67  # non-valid path, see https://bugs.pypy.org/issue1164
     68  except DistutilsPlatformError:
     69      STD_LIB_DIR = '//'
     70
     71  EXT_LIB_DIR = get_python_lib()
     72
     73  BUILTIN_MODULES = dict.fromkeys(sys.builtin_module_names, True)
     74
     75
     76 -class NoSourceFile(Exception):
     77      """exception raised when we are not able to get a python
     78      source file for a precompiled file
     79      """
     80
     81 -class LazyObject(object):
     82 -    def __init__(self, module, obj):
     83          self.module = module
     84          self.obj = obj
     85          self._imported = None
     86
     87 -    def _getobj(self):
     88          if self._imported is None:
     89             self._imported = getattr(load_module_from_name(self.module),
     90                                      self.obj)
     91          return self._imported
     92
     93 -    def __getattribute__(self, attr):
     94          try:
     95              return super(LazyObject, self).__getattribute__(attr)
     96          except AttributeError as ex:
     97              return getattr(self._getobj(), attr)
     98
     99 -    def __call__(self, *args, **kwargs):
    100          return self._getobj()(*args, **kwargs)
    101
    102
    103 -def load_module_from_name(dotted_name, path=None, use_sys=True):
    104      """Load a Python module from its name.
    105
    106      :type dotted_name: str
    107      :param dotted_name: python name of a module or package
    108
    109      :type path: list or None
    110      :param path:
    111        optional list of path where the module or package should be
    112        searched (use sys.path if nothing or None is given)
    113
    114      :type use_sys: bool
    115      :param use_sys:
    116        boolean indicating whether the sys.modules dictionary should be
    117        used or not
    118
    119
    120      :raise ImportError: if the module or package is not found
    121
    122      :rtype: module
    123      :return: the loaded module
    124      """
    125      return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
    126
    127
    128 -def load_module_from_modpath(parts, path=None, use_sys=True):
    129      """Load a python module from its splitted name.
    130
    131      :type parts: list(str) or tuple(str)
    132      :param parts:
    133        python name of a module or package splitted on '.'
    134
    135      :type path: list or None
    136      :param path:
    137        optional list of path where the module or package should be
    138        searched (use sys.path if nothing or None is given)
    139
    140      :type use_sys: bool
    141      :param use_sys:
    142        boolean indicating whether the sys.modules dictionary should be used or not
    143
    144      :raise ImportError: if the module or package is not found
    145
    146      :rtype: module
    147      :return: the loaded module
    148      """
    149      if use_sys:
    150          try:
    151              return sys.modules['.'.join(parts)]
    152          except KeyError:
    153              pass
    154      modpath = []
    155      prevmodule = None
    156      for part in parts:
    157          modpath.append(part)
    158          curname = '.'.join(modpath)
    159          module = None
    160          if len(modpath) != len(parts):
    161              # even with use_sys=False, should try to get outer packages from sys.modules
    162              module = sys.modules.get(curname)
    163          elif use_sys:
    164              # because it may have been indirectly loaded through a parent
    165              module = sys.modules.get(curname)
    166          if module is None:
    167              mp_file, mp_filename, mp_desc = find_module(part, path)
    168              try:
    169                  module = load_module(curname, mp_file, mp_filename, mp_desc)
    170              finally:
    171                  if mp_file is not None:
    172                      mp_file.close()
    173          if prevmodule:
    174              setattr(prevmodule, part, module)
    175          _file = getattr(module, '__file__', '')
    176          prevmodule = module
    177          if not _file and _is_namespace(curname):
    178              continue
    179          if not _file and len(modpath) != len(parts):
    180              raise ImportError('no module in %s' % '.'.join(parts[len(modpath):]) )
    181          path = [dirname( _file )]
    182      return module
    183
    184
    185 -def load_module_from_file(filepath, path=None, use_sys=True, extrapath=None):
    186      """Load a Python module from it's path.
    187
    188      :type filepath: str
    189      :param filepath: path to the python module or package
    190
    191      :type path: list or None
    192      :param path:
    193        optional list of path where the module or package should be
    194        searched (use sys.path if nothing or None is given)
    195
    196      :type use_sys: bool
    197      :param use_sys:
    198        boolean indicating whether the sys.modules dictionary should be
    199        used or not
    200
    201
    202      :raise ImportError: if the module or package is not found
    203
    204      :rtype: module
    205      :return: the loaded module
    206      """
    207      modpath = modpath_from_file(filepath, extrapath)
    208      return load_module_from_modpath(modpath, path, use_sys)
    209
    210
    211 -def _check_init(path, mod_path):
    212      """check there are some __init__.py all along the way"""
    213      modpath = []
    214      for part in mod_path:
    215          modpath.append(part)
    216          path = join(path, part)
    217          if not _is_namespace('.'.join(modpath)) and not _has_init(path):
    218              return False
    219      return True
    220
    221
    222 -def modpath_from_file(filename, extrapath=None):
    223      """given a file path return the corresponding splitted module's name
    224      (i.e name of a module or package splitted on '.')
    225
    226      :type filename: str
    227      :param filename: file's path for which we want the module's name
    228
    229      :type extrapath: dict
    230      :param extrapath:
    231        optional extra search path, with path as key and package name for the path
    232        as value. This is usually useful to handle package splitted in multiple
    233        directories using __path__ trick.
    234
    235
    236      :raise ImportError:
    237        if the corresponding module's name has not been found
    238
    239      :rtype: list(str)
    240      :return: the corresponding splitted module's name
    241      """
    242      base = splitext(abspath(filename))[0]
    243      if extrapath is not None:
    244          for path_ in extrapath:
    245              path = abspath(path_)
    246              if path and base[:len(path)] == path:
    247                  submodpath = [pkg for pkg in base[len(path):].split(os.sep)
    248                                if pkg]
    249                  if _check_init(path, submodpath[:-1]):
    250                      return extrapath[path_].split('.') + submodpath
    251      for path in sys.path:
    252          path = abspath(path)
    253          if path and base.startswith(path):
    254              modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg]
    255              if _check_init(path, modpath[:-1]):
    256                  return modpath
    257      raise ImportError('Unable to find module for %s in %s' % (
    258          filename, ', \n'.join(sys.path)))
    259
    260
    261
    262 -def file_from_modpath(modpath, path=None, context_file=None):
    263      """given a mod path (i.e. splitted module / package name), return the
    264      corresponding file, giving priority to source file over precompiled
    265      file if it exists
    266
    267      :type modpath: list or tuple
    268      :param modpath:
    269        splitted module's name (i.e name of a module or package splitted
    270        on '.')
    271        (this means explicit relative imports that start with dots have
    272        empty strings in this list!)
    273
    274      :type path: list or None
    275      :param path:
    276        optional list of path where the module or package should be
    277        searched (use sys.path if nothing or None is given)
    278
    279      :type context_file: str or None
    280      :param context_file:
    281        context file to consider, necessary if the identifier has been
    282        introduced using a relative import unresolvable in the actual
    283        context (i.e. modutils)
    284
    285      :raise ImportError: if there is no such module in the directory
    286
    287      :rtype: str or None
    288      :return:
    289        the path to the module's file or None if it's an integrated
    290        builtin module such as 'sys'
    291      """
    292      if context_file is not None:
    293          context = dirname(context_file)
    294      else:
    295          context = context_file
    296      if modpath[0] == 'xml':
    297          # handle _xmlplus
    298          try:
    299              return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context)
    300          except ImportError:
    301              return _file_from_modpath(modpath, path, context)
    302      elif modpath == ['os', 'path']:
    303          # FIXME: currently ignoring search_path...
    304          return os.path.__file__
    305      return _file_from_modpath(modpath, path, context)
    306
    307
    308
    309 -def get_module_part(dotted_name, context_file=None):
    310      """given a dotted name return the module part of the name :
    311
    312      >>> get_module_part('logilab.common.modutils.get_module_part')
    313      'logilab.common.modutils'
    314
    315      :type dotted_name: str
    316      :param dotted_name: full name of the identifier we are interested in
    317
    318      :type context_file: str or None
    319      :param context_file:
    320        context file to consider, necessary if the identifier has been
    321        introduced using a relative import unresolvable in the actual
    322        context (i.e. modutils)
    323
    324
    325      :raise ImportError: if there is no such module in the directory
    326
    327      :rtype: str or None
    328      :return:
    329        the module part of the name or None if we have not been able at
    330        all to import the given name
    331
    332      XXX: deprecated, since it doesn't handle package precedence over module
    333      (see #10066)
    334      """
    335      # os.path trick
    336      if dotted_name.startswith('os.path'):
    337          return 'os.path'
    338      parts = dotted_name.split('.')
    339      if context_file is not None:
    340          # first check for builtin module which won't be considered latter
    341          # in that case (path != None)
    342          if parts[0] in BUILTIN_MODULES:
    343              if len(parts) > 2:
    344                  raise ImportError(dotted_name)
    345              return parts[0]
    346          # don't use += or insert, we want a new list to be created !
    347      path = None
    348      starti = 0
    349      if parts[0] == '':
    350          assert context_file is not None, \
    351                  'explicit relative import, but no context_file?'
    352          path = [] # prevent resolving the import non-relatively
    353          starti = 1
    354      while parts[starti] == '': # for all further dots: change context
    355          starti += 1
    356          context_file = dirname(context_file)
    357      for i in range(starti, len(parts)):
    358          try:
    359              file_from_modpath(parts[starti:i+1],
    360                      path=path, context_file=context_file)
    361          except ImportError:
    362              if not i >= max(1, len(parts) - 2):
    363                  raise
    364              return '.'.join(parts[:i])
    365      return dotted_name
    366
    367
    368 -def get_modules(package, src_directory, blacklist=STD_BLACKLIST):
    369      """given a package directory return a list of all available python
    370      modules in the package and its subpackages
    371
    372      :type package: str
    373      :param package: the python name for the package
    374
    375      :type src_directory: str
    376      :param src_directory:
    377        path of the directory corresponding to the package
    378
    379      :type blacklist: list or tuple
    380      :param blacklist:
    381        optional list of files or directory to ignore, default to
    382        the value of `logilab.common.STD_BLACKLIST`
    383
    384      :rtype: list
    385      :return:
    386        the list of all available python modules in the package and its
    387        subpackages
    388      """
    389      modules = []
    390      for directory, dirnames, filenames in os.walk(src_directory):
    391          _handle_blacklist(blacklist, dirnames, filenames)
    392          # check for __init__.py
    393          if not '__init__.py' in filenames:
    394              dirnames[:] = ()
    395              continue
    396          if directory != src_directory:
    397              dir_package = directory[len(src_directory):].replace(os.sep, '.')
    398              modules.append(package + dir_package)
    399          for filename in filenames:
    400              if _is_python_file(filename) and filename != '__init__.py':
    401                  src = join(directory, filename)
    402                  module = package + src[len(src_directory):-3]
    403                  modules.append(module.replace(os.sep, '.'))
    404      return modules
    405
    406
    407
    408 -def get_module_files(src_directory, blacklist=STD_BLACKLIST):
    409      """given a package directory return a list of all available python
    410      module's files in the package and its subpackages
    411
    412      :type src_directory: str
    413      :param src_directory:
    414        path of the directory corresponding to the package
    415
    416      :type blacklist: list or tuple
    417      :param blacklist:
    418        optional list of files or directory to ignore, default to the value of
    419        `logilab.common.STD_BLACKLIST`
    420
    421      :rtype: list
    422      :return:
    423        the list of all available python module's files in the package and
    424        its subpackages
    425      """
    426      files = []
    427      for directory, dirnames, filenames in os.walk(src_directory):
    428          _handle_blacklist(blacklist, dirnames, filenames)
    429          # check for __init__.py
    430          if not '__init__.py' in filenames:
    431              dirnames[:] = ()
    432              continue
    433          for filename in filenames:
    434              if _is_python_file(filename):
    435                  src = join(directory, filename)
    436                  files.append(src)
    437      return files
    438
    439
    440 -def get_source_file(filename, include_no_ext=False):
    441      """given a python module's file name return the matching source file
    442      name (the filename will be returned identically if it's a already an
    443      absolute path to a python source file...)
    444
    445      :type filename: str
    446      :param filename: python module's file name
    447
    448
    449      :raise NoSourceFile: if no source file exists on the file system
    450
    451      :rtype: str
    452      :return: the absolute path of the source file if it exists
    453      """
    454      base, orig_ext = splitext(abspath(filename))
    455      for ext in PY_SOURCE_EXTS:
    456          source_path = '%s.%s' % (base, ext)
    457          if exists(source_path):
    458              return source_path
    459      if include_no_ext and not orig_ext and exists(base):
    460          return base
    461      raise NoSourceFile(filename)
    462
    463
    464 -def cleanup_sys_modules(directories):
    465      """remove submodules of `directories` from `sys.modules`"""
    466      cleaned = []
    467      for modname, module in list(sys.modules.items()):
    468          modfile = getattr(module, '__file__', None)
    469          if modfile:
    470              for directory in directories:
    471                  if modfile.startswith(directory):
    472                      cleaned.append(modname)
    473                      del sys.modules[modname]
    474                      break
    475      return cleaned
    476
    477
    478 -def is_python_source(filename):
    479      """
    480      rtype: bool
    481      return: True if the filename is a python source file
    482      """
    483      return splitext(filename)[1][1:] in PY_SOURCE_EXTS
    484
    485
    486 -def is_standard_module(modname, std_path=(STD_LIB_DIR,)):
    487      """try to guess if a module is a standard python module (by default,
    488      see `std_path` parameter's description)
    489
    490      :type modname: str
    491      :param modname: name of the module we are interested in
    492
    493      :type std_path: list(str) or tuple(str)
    494      :param std_path: list of path considered as standard
    495
    496
    497      :rtype: bool
    498      :return:
    499        true if the module:
    500        - is located on the path listed in one of the directory in `std_path`
    501        - is a built-in module
    502
    503      Note: this function is known to return wrong values when inside virtualenv.
    504      See https://www.logilab.org/ticket/294756.
    505      """
    506      modname = modname.split('.')[0]
    507      try:
    508          filename = file_from_modpath([modname])
    509      except ImportError as ex:
    510          # import failed, i'm probably not so wrong by supposing it's
    511          # not standard...
    512          return False
    513      # modules which are not living in a file are considered standard
    514      # (sys and __builtin__ for instance)
    515      if filename is None:
    516          # we assume there are no namespaces in stdlib
    517          return not _is_namespace(modname)
    518      filename = abspath(filename)
    519      if filename.startswith(EXT_LIB_DIR):
    520          return False
    521      for path in std_path:
    522          if filename.startswith(abspath(path)):
    523              return True
    524      return False
    525
    526
    527
    528 -def is_relative(modname, from_file):
    529      """return true if the given module name is relative to the given
    530      file name
    531
    532      :type modname: str
    533      :param modname: name of the module we are interested in
    534
    535      :type from_file: str
    536      :param from_file:
    537        path of the module from which modname has been imported
    538
    539      :rtype: bool
    540      :return:
    541        true if the module has been imported relatively to `from_file`
    542      """
    543      if not isdir(from_file):
    544          from_file = dirname(from_file)
    545      if from_file in sys.path:
    546          return False
    547      try:
    548          find_module(modname.split('.')[0], [from_file])
    549          return True
    550      except ImportError:
    551          return False
    552
    553
    554  # internal only functions #####################################################
    555
    556 -def _file_from_modpath(modpath, path=None, context=None):
    557      """given a mod path (i.e. splitted module / package name), return the
    558      corresponding file
    559
    560      this function is used internally, see `file_from_modpath`'s
    561      documentation for more information
    562      """
    563      assert len(modpath) > 0
    564      if context is not None:
    565          try:
    566              mtype, mp_filename = _module_file(modpath, [context])
    567          except ImportError:
    568              mtype, mp_filename = _module_file(modpath, path)
    569      else:
    570          mtype, mp_filename = _module_file(modpath, path)
    571      if mtype == PY_COMPILED:
    572          try:
    573              return get_source_file(mp_filename)
    574          except NoSourceFile:
    575              return mp_filename
    576      elif mtype == C_BUILTIN:
    577          # integrated builtin module
    578          return None
    579      elif mtype == PKG_DIRECTORY:
    580          mp_filename = _has_init(mp_filename)
    581      return mp_filename
    582
    583 -def _search_zip(modpath, pic):
    584      for filepath, importer in pic.items():
    585          if importer is not None:
    586              if importer.find_module(modpath[0]):
    587                  if not importer.find_module('/'.join(modpath)):
    588                      raise ImportError('No module named %s in %s/%s' % (
    589                          '.'.join(modpath[1:]), filepath, modpath))
    590                  return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath
    591      raise ImportError('No module named %s' % '.'.join(modpath))
    592
    593  try:
    594      import pkg_resources
    595  except ImportError:
    596      pkg_resources = None
    597
    598
    599 -def _is_namespace(modname):
    600      return (pkg_resources is not None
    601              and modname in pkg_resources._namespace_packages)
    602
    603
    604 -def _module_file(modpath, path=None):
    605      """get a module type / file path
    606
    607      :type modpath: list or tuple
    608      :param modpath:
    609        splitted module's name (i.e name of a module or package splitted
    610        on '.'), with leading empty strings for explicit relative import
    611
    612      :type path: list or None
    613      :param path:
    614        optional list of path where the module or package should be
    615        searched (use sys.path if nothing or None is given)
    616
    617
    618      :rtype: tuple(int, str)
    619      :return: the module type flag and the file path for a module
    620      """
    621      # egg support compat
    622      try:
    623          pic = sys.path_importer_cache
    624          _path = (path is None and sys.path or path)
    625          for __path in _path:
    626              if not __path in pic:
    627                  try:
    628                      pic[__path] = zipimport.zipimporter(__path)
    629                  except zipimport.ZipImportError:
    630                      pic[__path] = None
    631          checkeggs = True
    632      except AttributeError:
    633          checkeggs = False
    634      # pkg_resources support (aka setuptools namespace packages)
    635      if (_is_namespace(modpath[0]) and modpath[0] in sys.modules):
    636          # setuptools has added into sys.modules a module object with proper
    637          # __path__, get back information from there
    638          module = sys.modules[modpath.pop(0)]
    639          path = module.__path__
    640          if not modpath:
    641              return C_BUILTIN, None
    642      imported = []
    643      while modpath:
    644          modname = modpath[0]
    645          # take care to changes in find_module implementation wrt builtin modules
    646          #
    647          # Python 2.6.6 (r266:84292, Sep 11 2012, 08:34:23)
    648          # >>> imp.find_module('posix')
    649          # (None, 'posix', ('', '', 6))
    650          #
    651          # Python 3.3.1 (default, Apr 26 2013, 12:08:46)
    652          # >>> imp.find_module('posix')
    653          # (None, None, ('', '', 6))
    654          try:
    655              _, mp_filename, mp_desc = find_module(modname, path)
    656          except ImportError:
    657              if checkeggs:
    658                  return _search_zip(modpath, pic)[:2]
    659              raise
    660          else:
    661              if checkeggs and mp_filename:
    662                  fullabspath = [abspath(x) for x in _path]
    663                  try:
    664                      pathindex = fullabspath.index(dirname(abspath(mp_filename)))
    665                      emtype, emp_filename, zippath = _search_zip(modpath, pic)
    666                      if pathindex > _path.index(zippath):
    667                          # an egg takes priority
    668                          return emtype, emp_filename
    669                  except ValueError:
    670                      # XXX not in _path
    671                      pass
    672                  except ImportError:
    673                      pass
    674                  checkeggs = False
    675          imported.append(modpath.pop(0))
    676          mtype = mp_desc[2]
    677          if modpath:
    678              if mtype != PKG_DIRECTORY:
    679                  raise ImportError('No module %s in %s' % ('.'.join(modpath),
    680                                                            '.'.join(imported)))
    681              # XXX guess if package is using pkgutil.extend_path by looking for
    682              # those keywords in the first four Kbytes
    683              try:
    684                  with open(join(mp_filename, '__init__.py')) as stream:
    685                      data = stream.read(4096)
    686              except IOError:
    687                  path = [mp_filename]
    688              else:
    689                  if 'pkgutil' in data and 'extend_path' in data:
    690                      # extend_path is called, search sys.path for module/packages
    691                      # of this name see pkgutil.extend_path documentation
    692                      path = [join(p, *imported) for p in sys.path
    693                              if isdir(join(p, *imported))]
    694                  else:
    695                      path = [mp_filename]
    696      return mtype, mp_filename
    697
    698 -def _is_python_file(filename):
    699      """return true if the given filename should be considered as a python file
    700
    701      .pyc and .pyo are ignored
    702      """
    703      for ext in ('.py', '.so', '.pyd', '.pyw'):
    704          if filename.endswith(ext):
    705              return True
    706      return False
    707
    708
    709 -def _has_init(directory):
    710      """if the given directory has a valid __init__ file, return its path,
    711      else return None
    712      """
    713      mod_or_pack = join(directory, '__init__')
    714      for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'):
    715          if exists(mod_or_pack + '.' + ext):
    716              return mod_or_pack + '.' + ext
    717      return None
    718