Package web2py :: Package gluon :: Module fileutils

[hide private]
[frames] | no frames]

# Source Code for Module web2py.gluon.fileutils

      1  #!/usr/bin/env python
      2  # -*- coding: utf-8 -*-
      3
      4  """
      5  This file is part of the web2py Web Framework
      6  Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
      7  License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
      8  """
      9
     10  import storage
     11  import os
     12  import re
     13  import tarfile
     14  import glob
     15  import time
     16  import datetime
     17  from http import HTTP
     18  from gzip import open as gzopen
     19  from settings import global_settings
     20
     21
     22  __all__ = [
     23      'parse_version',
     24      'read_file',
     25      'write_file',
     26      'readlines_file',
     27      'up',
     28      'abspath',
     29      'mktree',
     30      'listdir',
     31      'recursive_unlink',
     32      'cleanpath',
     33      'tar',
     34      'untar',
     35      'tar_compiled',
     36      'get_session',
     37      'check_credentials',
     38      'w2p_pack',
     39      'w2p_unpack',
     40      'w2p_pack_plugin',
     41      'w2p_unpack_plugin',
     42      'fix_newlines',
     43      'make_fake_file_like_object',
     44      ]
     45
     46 -def parse_version(version = "Version 1.99.0 (2011-09-19 08:23:26)"):
     47      re_version = re.compile('[^\d]+ (\d+)\.(\d+)\.(\d+)\s*\((?P<datetime>.+?)\)\s*(?P<type>[a-z]+)?')
     48      m = re_version.match(version)
     49      a,b,c = int(m.group(1)),int(m.group(2)),int(m.group(3)),
     50      s = m.group('type') or 'dev'
     51      d = datetime.datetime.strptime(m.group('datetime'),'%Y-%m-%d %H:%M:%S')
     52      return (a,b,c,d,s)
     53
     54 -def read_file(filename, mode='r'):
     55      "returns content from filename, making sure to close the file explicitly on exit."
     56      f = open(filename, mode)
     57      try:
     58          return f.read()
     59      finally:
     60          f.close()
     61
     62 -def write_file(filename, value, mode='w'):
     63      "writes <value> to filename, making sure to close the file explicitly on exit."
     64      f = open(filename, mode)
     65      try:
     66          return f.write(value)
     67      finally:
     68          f.close()
     69
     70 -def readlines_file(filename, mode='r'):
     71      "applies .split('\n') to the output of read_file()"
     72      return read_file(filename, mode).split('\n')
     73
     74 -def abspath(*relpath, **base):
     75      "convert relative path to absolute path based (by default) on applications_parent"
     76      path = os.path.join(*relpath)
     77      gluon = base.get('gluon', False)
     78      if os.path.isabs(path):
     79          return path
     80      if gluon:
     81          return os.path.join(global_settings.gluon_parent, path)
     82      return os.path.join(global_settings.applications_parent, path)
     83
     84
     85 -def mktree(path):
     86      head,tail =os.path.split(path)
     87      if head:
     88          if tail: mktree(head)
     89          if not os.path.exists(head):
     90              os.mkdir(head)
     91
     92 -def listdir(
     93      path,
     94      expression='^.+$',
     95      drop=True,
     96      add_dirs=False,
     97      sort=True,
     98      ):
     99      """
    100      like os.listdir() but you can specify a regex pattern to filter files.
    101      if add_dirs is True, the returned items will have the full path.
    102      """
    103      if path[-1:] != os.path.sep:
    104          path = path + os.path.sep
    105      if drop:
    106          n = len(path)
    107      else:
    108          n = 0
    109      regex = re.compile(expression)
    110      items = []
    111      for (root, dirs, files) in os.walk(path, topdown=True):
    112          for dir in dirs[:]:
    113              if dir.startswith('.'):
    114                  dirs.remove(dir)
    115          if add_dirs:
    116              items.append(root[n:])
    117          for file in sorted(files):
    118              if regex.match(file) and not file.startswith('.'):
    119                  items.append(os.path.join(root, file)[n:])
    120      if sort:
    121          return sorted(items)
    122      else:
    123          return items
    124
    125
    126 -def recursive_unlink(f):
    127      if os.path.isdir(f):
    128          for s in os.listdir(f):
    129              recursive_unlink(os.path.join(f,s))
    130          os.rmdir(f)
    131      elif os.path.isfile(f):
    132          os.unlink(f)
    133
    134
    135 -def cleanpath(path):
    136      """
    137      turns any expression/path into a valid filename. replaces / with _ and
    138      removes special characters.
    139      """
    140
    141      items = path.split('.')
    142      if len(items) > 1:
    143          path = re.sub('[^\w\.]+', '_', '_'.join(items[:-1]) + '.'
    144                         + ''.join(items[-1:]))
    145      else:
    146          path = re.sub('[^\w\.]+', '_', ''.join(items[-1:]))
    147      return path
    148
    149
    150 -def _extractall(filename, path='.', members=None):
    151      if not hasattr(tarfile.TarFile, 'extractall'):
    152          from tarfile import ExtractError
    153
    154          class TarFile(tarfile.TarFile):
    155
    156              def extractall(self, path='.', members=None):
    157                  """Extract all members from the archive to the current working
    158               directory and set owner, modification time and permissions on
    159               directories afterwards. `path' specifies a different directory
    160               to extract to. `members' is optional and must be a subset of the
    161               list returned by getmembers().
    162                  """
    163
    164                  directories = []
    165                  if members is None:
    166                      members = self
    167                  for tarinfo in members:
    168                      if tarinfo.isdir():
    169
    170                          # Extract directory with a safe mode, so that
    171                          # all files below can be extracted as well.
    172
    173                          try:
    174                              os.makedirs(os.path.join(path,
    175                                      tarinfo.name), 0777)
    176                          except EnvironmentError:
    177                              pass
    178                          directories.append(tarinfo)
    179                      else:
    180                          self.extract(tarinfo, path)
    181
    182                  # Reverse sort directories.
    183
    184                  directories.sort(lambda a, b: cmp(a.name, b.name))
    185                  directories.reverse()
    186
    187                  # Set correct owner, mtime and filemode on directories.
    188
    189                  for tarinfo in directories:
    190                      path = os.path.join(path, tarinfo.name)
    191                      try:
    192                          self.chown(tarinfo, path)
    193                          self.utime(tarinfo, path)
    194                          self.chmod(tarinfo, path)
    195                      except ExtractError, e:
    196                          if self.errorlevel > 1:
    197                              raise
    198                          else:
    199                              self._dbg(1, 'tarfile: %s' % e)
    200
    201
    202          _cls = TarFile
    203      else:
    204          _cls = tarfile.TarFile
    205
    206      tar = _cls(filename, 'r')
    207      ret = tar.extractall(path, members)
    208      tar.close()
    209      return ret
    210
    211 -def tar(file, dir, expression='^.+$'):
    212      """
    213      tars dir into file, only tars file that match expression
    214      """
    215
    216      tar = tarfile.TarFile(file, 'w')
    217      try:
    218          for file in listdir(dir, expression, add_dirs=True):
    219              tar.add(os.path.join(dir, file), file, False)
    220      finally:
    221          tar.close()
    222
    223 -def untar(file, dir):
    224      """
    225      untar file into dir
    226      """
    227
    228      _extractall(file, dir)
    229
    230
    231 -def w2p_pack(filename, path, compiled=False):
    232      filename = abspath(filename)
    233      path = abspath(path)
    234      tarname = filename + '.tar'
    235      if compiled:
    236          tar_compiled(tarname, path, '^[\w\.\-]+$')
    237      else:
    238          tar(tarname, path, '^[\w\.\-]+$')
    239      w2pfp = gzopen(filename, 'wb')
    240      tarfp = open(tarname, 'rb')
    241      w2pfp.write(tarfp.read())
    242      w2pfp.close()
    243      tarfp.close()
    244      os.unlink(tarname)
    245
    246 -def w2p_unpack(filename, path, delete_tar=True):
    247      filename = abspath(filename)
    248      path = abspath(path)
    249      if filename[-4:] == '.w2p' or filename[-3:] == '.gz':
    250          if filename[-4:] == '.w2p':
    251              tarname = filename[:-4] + '.tar'
    252          else:
    253              tarname = filename[:-3] + '.tar'
    254          fgzipped = gzopen(filename, 'rb')
    255          tarfile = open(tarname, 'wb')
    256          tarfile.write(fgzipped.read())
    257          tarfile.close()
    258          fgzipped.close()
    259      else:
    260          tarname = filename
    261      untar(tarname, path)
    262      if delete_tar:
    263          os.unlink(tarname)
    264
    265
    266 -def w2p_pack_plugin(filename, path, plugin_name):
    267      """Pack the given plugin into a w2p file.
    268      Will match files at:
    269          <path>/*/plugin_[name].*
    270          <path>/*/plugin_[name]/*
    271      """
    272      filename = abspath(filename)
    273      path = abspath(path)
    274      if not filename.endswith('web2py.plugin.%s.w2p' % plugin_name):
    275          raise Exception, "Not a web2py plugin name"
    276      plugin_tarball = tarfile.open(filename, 'w:gz')
    277      try:
    278          app_dir = path
    279          while app_dir[-1]=='/':
    280              app_dir = app_dir[:-1]
    281          files1=glob.glob(os.path.join(app_dir,'*/plugin_%s.*' % plugin_name))
    282          files2=glob.glob(os.path.join(app_dir,'*/plugin_%s/*' % plugin_name))
    283          for file in files1+files2:
    284              plugin_tarball.add(file, arcname=file[len(app_dir)+1:])
    285      finally:
    286          plugin_tarball.close()
    287
    288
    289 -def w2p_unpack_plugin(filename, path, delete_tar=True):
    290      filename = abspath(filename)
    291      path = abspath(path)
    292      if not os.path.basename(filename).startswith('web2py.plugin.'):
    293          raise Exception, "Not a web2py plugin"
    294      w2p_unpack(filename,path,delete_tar)
    295
    296
    297 -def tar_compiled(file, dir, expression='^.+$'):
    298      """
    299      used to tar a compiled application.
    300      the content of models, views, controllers is not stored in the tar file.
    301      """
    302
    303      tar = tarfile.TarFile(file, 'w')
    304      for file in listdir(dir, expression, add_dirs=True):
    305          filename = os.path.join(dir, file)
    306          if os.path.islink(filename):
    307              continue
    308          if os.path.isfile(filename) and file[-4:] != '.pyc':
    309              if file[:6] == 'models':
    310                  continue
    311              if file[:5] == 'views':
    312                  continue
    313              if file[:11] == 'controllers':
    314                  continue
    315              if file[:7] == 'modules':
    316                  continue
    317          tar.add(filename, file, False)
    318      tar.close()
    319
    320 -def up(path):
    321      return os.path.dirname(os.path.normpath(path))
    322
    323
    324 -def get_session(request, other_application='admin'):
    325      """ checks that user is authorized to access other_application"""
    326      if request.application == other_application:
    327          raise KeyError
    328      try:
    329          session_id = request.cookies['session_id_' + other_application].value
    330          osession = storage.load_storage(os.path.join(
    331                  up(request.folder), other_application, 'sessions', session_id))
    332      except:
    333          osession = storage.Storage()
    334      return osession
    335
    336
    337 -def check_credentials(request, other_application='admin', expiration = 60*60):
    338      """ checks that user is authorized to access other_application"""
    339      if request.env.web2py_runtime_gae:
    340          from google.appengine.api import users
    341          if users.is_current_user_admin():
    342              return True
    343          else:
    344              login_html = '<a href="%s">Sign in with your google account</a>.' \
    345                  % users.create_login_url(request.env.path_info)
    346              raise HTTP(200, '<html><body>%s</body></html>' % login_html)
    347      else:
    348          dt = time.time() - expiration
    349          s = get_session(request, other_application)
    350          return (s.authorized and s.last_time and s.last_time > dt)
    351
    352
    353 -def fix_newlines(path):
    354      regex = re.compile(r'''(\r
    355  |\r|
    356  )''')
    357      for filename in listdir(path, '.*\.(py|html)$', drop=False):
    358          rdata = read_file(filename, 'rb')
    359          wdata = regex.sub('\n', rdata)
    360          if wdata != rdata:
    361              write_file(filename, wdata, 'wb')
    362
    363 -def copystream(
    364      src,
    365      dest,
    366      size,
    367      chunk_size=10 ** 5,
    368      ):
    369      """
    370      this is here because I think there is a bug in shutil.copyfileobj
    371      """
    372      while size > 0:
    373          if size < chunk_size:
    374              data = src.read(size)
    375          else:
    376              data = src.read(chunk_size)
    377          length = len(data)
    378          if length > size:
    379              (data, length) = (data[:size], size)
    380          size -= length
    381          if length == 0:
    382              break
    383          dest.write(data)
    384          if length < chunk_size:
    385              break
    386      dest.seek(0)
    387      return
    388
    389
    390 -def make_fake_file_like_object():
    391      class LogFile(object):
    392          def write(self, value):
    393              pass
    394          def close(self):
    395              pass
    396      return LogFile()
    397