Package logilab :: Package common :: Module fileutils

[frames] | no frames]

# Source Code for Module logilab.common.fileutils

      1  # copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
      2  # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
      3  #
      4  # This file is part of logilab-common.
      5  #
      6  # logilab-common is free software: you can redistribute it and/or modify it under
      7  # the terms of the GNU Lesser General Public License as published by the Free
      8  # Software Foundation, either version 2.1 of the License, or (at your option) any
      9  # later version.
     10  #
     11  # logilab-common is distributed in the hope that it will be useful, but WITHOUT
     12  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
     13  # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
     14  # details.
     15  #
     16  # You should have received a copy of the GNU Lesser General Public License along
     17  # with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
     18  """File and file-path manipulation utilities.
     19
     20  :group path manipulation: first_level_directory, relative_path, is_binary,\
     21  get_by_ext, remove_dead_links
     22  :group file manipulation: norm_read, norm_open, lines, stream_lines, lines,\
     23  write_open_mode, ensure_fs_mode, export
     24  :sort: path manipulation, file manipulation
     25  """
     26
     27  from __future__ import print_function
     28
     29  __docformat__ = "restructuredtext en"
     30
     31  import io
     32  import sys
     33  import shutil
     34  import mimetypes
     35  from os.path import isabs, isdir, islink, split, exists, normpath, join
     36  from os.path import abspath
     37  from os import sep, mkdir, remove, listdir, stat, chmod, walk
     38  from stat import ST_MODE, S_IWRITE
     39
     40  from logilab.common import STD_BLACKLIST as BASE_BLACKLIST, IGNORED_EXTENSIONS
     41  from logilab.common.shellutils import find
     42  from logilab.common.deprecation import deprecated
     43  from logilab.common.compat import FileIO
     44
     45 -def first_level_directory(path):
     46      """Return the first level directory of a path.
     47
     48      >>> first_level_directory('home/syt/work')
     49      'home'
     50      >>> first_level_directory('/home/syt/work')
     51      '/'
     52      >>> first_level_directory('work')
     53      'work'
     54      >>>
     55
     56      :type path: str
     57      :param path: the path for which we want the first level directory
     58
     59      :rtype: str
     60      :return: the first level directory appearing in `path`
     61      """
     62      head, tail = split(path)
     63      while head and tail:
     64          head, tail = split(head)
     65      if tail:
     66          return tail
     67      # path was absolute, head is the fs root
     68      return head
     69
     70 -def abspath_listdir(path):
     71      """Lists path's content using absolute paths."""
     72      path = abspath(path)
     73      return [join(path, filename) for filename in listdir(path)]
     74
     75
     76 -def is_binary(filename):
     77      """Return true if filename may be a binary file, according to it's
     78      extension.
     79
     80      :type filename: str
     81      :param filename: the name of the file
     82
     83      :rtype: bool
     84      :return:
     85        true if the file is a binary file (actually if it's mime type
     86        isn't beginning by text/)
     87      """
     88      try:
     89          return not mimetypes.guess_type(filename)[0].startswith('text')
     90      except AttributeError:
     91          return 1
     92
     93
     94 -def write_open_mode(filename):
     95      """Return the write mode that should used to open file.
     96
     97      :type filename: str
     98      :param filename: the name of the file
     99
    100      :rtype: str
    101      :return: the mode that should be use to open the file ('w' or 'wb')
    102      """
    103      if is_binary(filename):
    104          return 'wb'
    105      return 'w'
    106
    107
    108 -def ensure_fs_mode(filepath, desired_mode=S_IWRITE):
    109      """Check that the given file has the given mode(s) set, else try to
    110      set it.
    111
    112      :type filepath: str
    113      :param filepath: path of the file
    114
    115      :type desired_mode: int
    116      :param desired_mode:
    117        ORed flags describing the desired mode. Use constants from the
    118        `stat` module for file permission's modes
    119      """
    120      mode = stat(filepath)[ST_MODE]
    121      if not mode & desired_mode:
    122          chmod(filepath, mode | desired_mode)
    123
    124
    125  # XXX (syt) unused? kill?
    126 -class ProtectedFile(FileIO):
    127      """A special file-object class that automatically does a 'chmod +w' when
    128      needed.
    129
    130      XXX: for now, the way it is done allows 'normal file-objects' to be
    131      created during the ProtectedFile object lifetime.
    132      One way to circumvent this would be to chmod / unchmod on each
    133      write operation.
    134
    135      One other way would be to :
    136
    137      - catch the IOError in the __init__
    138
    139      - if IOError, then create a StringIO object
    140
    141      - each write operation writes in this StringIO object
    142
    143      - on close()/del(), write/append the StringIO content to the file and
    144        do the chmod only once
    145      """
    146 -    def __init__(self, filepath, mode):
    147          self.original_mode = stat(filepath)[ST_MODE]
    148          self.mode_changed = False
    149          if mode in ('w', 'a', 'wb', 'ab'):
    150              if not self.original_mode & S_IWRITE:
    151                  chmod(filepath, self.original_mode | S_IWRITE)
    152                  self.mode_changed = True
    153          FileIO.__init__(self, filepath, mode)
    154
    155 -    def _restore_mode(self):
    156          """restores the original mode if needed"""
    157          if self.mode_changed:
    158              chmod(self.name, self.original_mode)
    159              # Don't re-chmod in case of several restore
    160              self.mode_changed = False
    161
    162 -    def close(self):
    163          """restore mode before closing"""
    164          self._restore_mode()
    165          FileIO.close(self)
    166
    167 -    def __del__(self):
    168          if not self.closed:
    169              self.close()
    170
    171
    172 -class UnresolvableError(Exception):
    173      """Exception raised by relative path when it's unable to compute relative
    174      path between two paths.
    175      """
    176
    177 -def relative_path(from_file, to_file):
    178      """Try to get a relative path from `from_file` to `to_file`
    179      (path will be absolute if to_file is an absolute file). This function
    180      is useful to create link in `from_file` to `to_file`. This typical use
    181      case is used in this function description.
    182
    183      If both files are relative, they're expected to be relative to the same
    184      directory.
    185
    186      >>> relative_path( from_file='toto/index.html', to_file='index.html')
    187      '../index.html'
    188      >>> relative_path( from_file='index.html', to_file='toto/index.html')
    189      'toto/index.html'
    190      >>> relative_path( from_file='tutu/index.html', to_file='toto/index.html')
    191      '../toto/index.html'
    192      >>> relative_path( from_file='toto/index.html', to_file='/index.html')
    193      '/index.html'
    194      >>> relative_path( from_file='/toto/index.html', to_file='/index.html')
    195      '../index.html'
    196      >>> relative_path( from_file='/toto/index.html', to_file='/toto/summary.html')
    197      'summary.html'
    198      >>> relative_path( from_file='index.html', to_file='index.html')
    199      ''
    200      >>> relative_path( from_file='/index.html', to_file='toto/index.html')
    201      Traceback (most recent call last):
    202        File "<string>", line 1, in ?
    203        File "<stdin>", line 37, in relative_path
    204      UnresolvableError
    205      >>> relative_path( from_file='/index.html', to_file='/index.html')
    206      ''
    207      >>>
    208
    209      :type from_file: str
    210      :param from_file: source file (where links will be inserted)
    211
    212      :type to_file: str
    213      :param to_file: target file (on which links point)
    214
    215      :raise UnresolvableError: if it has been unable to guess a correct path
    216
    217      :rtype: str
    218      :return: the relative path of `to_file` from `from_file`
    219      """
    220      from_file = normpath(from_file)
    221      to_file = normpath(to_file)
    222      if from_file == to_file:
    223          return ''
    224      if isabs(to_file):
    225          if not isabs(from_file):
    226              return to_file
    227      elif isabs(from_file):
    228          raise UnresolvableError()
    229      from_parts = from_file.split(sep)
    230      to_parts = to_file.split(sep)
    231      idem = 1
    232      result = []
    233      while len(from_parts) > 1:
    234          dirname = from_parts.pop(0)
    235          if idem and len(to_parts) > 1 and dirname == to_parts[0]:
    236              to_parts.pop(0)
    237          else:
    238              idem = 0
    239              result.append('..')
    240      result += to_parts
    241      return sep.join(result)
    242
    243
    244 -def norm_read(path):
    245      """Return the content of the file with normalized line feeds.
    246
    247      :type path: str
    248      :param path: path to the file to read
    249
    250      :rtype: str
    251      :return: the content of the file with normalized line feeds
    252      """
    253      return open(path, 'U').read()
    254  norm_read = deprecated("use \"open(path, 'U').read()\"")(norm_read)
    255
    256 -def norm_open(path):
    257      """Return a stream for a file with content with normalized line feeds.
    258
    259      :type path: str
    260      :param path: path to the file to open
    261
    262      :rtype: file or StringIO
    263      :return: the opened file with normalized line feeds
    264      """
    265      return open(path, 'U')
    266  norm_open = deprecated("use \"open(path, 'U')\"")(norm_open)
    267
    268 -def lines(path, comments=None):
    269      """Return a list of non empty lines in the file located at `path`.
    270
    271      :type path: str
    272      :param path: path to the file
    273
    274      :type comments: str or None
    275      :param comments:
    276        optional string which can be used to comment a line in the file
    277        (i.e. lines starting with this string won't be returned)
    278
    279      :rtype: list
    280      :return:
    281        a list of stripped line in the file, without empty and commented
    282        lines
    283
    284      :warning: at some point this function will probably return an iterator
    285      """
    286      with io.open(path) as stream:
    287          return stream_lines(stream, comments)
    288
    289
    290 -def stream_lines(stream, comments=None):
    291      """Return a list of non empty lines in the given `stream`.
    292
    293      :type stream: object implementing 'xreadlines' or 'readlines'
    294      :param stream: file like object
    295
    296      :type comments: str or None
    297      :param comments:
    298        optional string which can be used to comment a line in the file
    299        (i.e. lines starting with this string won't be returned)
    300
    301      :rtype: list
    302      :return:
    303        a list of stripped line in the file, without empty and commented
    304        lines
    305
    306      :warning: at some point this function will probably return an iterator
    307      """
    308      try:
    309          readlines = stream.xreadlines
    310      except AttributeError:
    311          readlines = stream.readlines
    312      result = []
    313      for line in readlines():
    314          line = line.strip()
    315          if line and (comments is None or not line.startswith(comments)):
    316              result.append(line)
    317      return result
    318
    319
    320 -def export(from_dir, to_dir,
    321             blacklist=BASE_BLACKLIST, ignore_ext=IGNORED_EXTENSIONS,
    322             verbose=0):
    323      """Make a mirror of `from_dir` in `to_dir`, omitting directories and
    324      files listed in the black list or ending with one of the given
    325      extensions.
    326
    327      :type from_dir: str
    328      :param from_dir: directory to export
    329
    330      :type to_dir: str
    331      :param to_dir: destination directory
    332
    333      :type blacklist: list or tuple
    334      :param blacklist:
    335        list of files or directories to ignore, default to the content of
    336        `BASE_BLACKLIST`
    337
    338      :type ignore_ext: list or tuple
    339      :param ignore_ext:
    340        list of extensions to ignore, default to  the content of
    341        `IGNORED_EXTENSIONS`
    342
    343      :type verbose: bool
    344      :param verbose:
    345        flag indicating whether information about exported files should be
    346        printed to stderr, default to False
    347      """
    348      try:
    349          mkdir(to_dir)
    350      except OSError:
    351          pass # FIXME we should use "exists" if the point is about existing dir
    352               # else (permission problems?) shouldn't return / raise ?
    353      for directory, dirnames, filenames in walk(from_dir):
    354          for norecurs in blacklist:
    355              try:
    356                  dirnames.remove(norecurs)
    357              except ValueError:
    358                  continue
    359          for dirname in dirnames:
    360              src = join(directory, dirname)
    361              dest = to_dir + src[len(from_dir):]
    362              if isdir(src):
    363                  if not exists(dest):
    364                      mkdir(dest)
    365          for filename in filenames:
    366              # don't include binary files
    367              # endswith does not accept tuple in 2.4
    368              if any([filename.endswith(ext) for ext in ignore_ext]):
    369                  continue
    370              src = join(directory, filename)
    371              dest = to_dir + src[len(from_dir):]
    372              if verbose:
    373                  print(src, '->', dest, file=sys.stderr)
    374              if exists(dest):
    375                  remove(dest)
    376              shutil.copy2(src, dest)
    377
    378
    379 -def remove_dead_links(directory, verbose=0):
    380      """Recursively traverse directory and remove all dead links.
    381
    382      :type directory: str
    383      :param directory: directory to cleanup
    384
    385      :type verbose: bool
    386      :param verbose:
    387        flag indicating whether information about deleted links should be
    388        printed to stderr, default to False
    389      """
    390      for dirpath, dirname, filenames in walk(directory):
    391          for filename in dirnames + filenames:
    392              src = join(dirpath, filename)
    393              if islink(src) and not exists(src):
    394                  if verbose:
    395                      print('remove dead link', src)
    396                  remove(src)
    397