Package csb :: Module build

[frames] | no frames]

# Source Code for Module csb.build

      1  """
      2  CSB build related tools and programs.
      3
      4  When executed as a program, this module will run the CSB Build Console and
      5  build the source tree it belongs to. The source tree is added at the
      6  B{beginning} of sys.path to make sure that all subsequent imports from the
      7  Test and Doc consoles will import the right thing (think of multiple CSB
      8  packages installed on the same server).
      9
     10  Here is how to build, test and package the whole project::
     11
     12      $ hg clone https://hg.codeplex.com/csb CSB
     13      $ CSB/csb/build.py -o <output directory>
     14
     15  The Console can also be imported and instantiated as a regular Python class.
     16  In this case the Console again builds the source tree it is part of, but
     17  sys.path will remain intact. Therefore, the Console will assume that all
     18  modules currently in memory, as well as those that can be subsequently imported
     19  by the Console itself, belong to the same CSB package.
     20
     21  @note: The CSB build services no longer support the option to build external
     22         source trees.
     23  @see: [CSB 0000038]
     24  """
     25  from __future__ import print_function
     26
     27  import os
     28  import sys
     29  import getopt
     30  import traceback
     31  import compileall
     32
     33  if os.path.basename(__file__) == '__init__.py':
     34      PARENT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
     35  else:
     36      PARENT = os.path.abspath(os.path.dirname(__file__))
     37
     38  ROOT = 'csb'
     39  SOURCETREE = os.path.abspath(os.path.join(PARENT, ".."))
     40
     41  if __name__ == '__main__':
     42
     43      # make sure "import io" imports the built in module, not csb.io
     44      # io is required by tarfile
     45      for path in sys.path:
     46          if path.startswith(SOURCETREE):
     47              sys.path.remove(path)
     48
     49      import io
     50      assert hasattr(io, 'BufferedIOBase')
     51
     52      sys.path = [SOURCETREE] + sys.path
     53
     54
     55  """
     56  It is now safe to import any modules
     57  """
     58  import imp
     59  import shutil
     60  import tarfile
     61
     62  import csb
     63
     64  from abc import ABCMeta, abstractmethod
     65  from csb.io import Shell
     66
     67
     68 -class BuildTypes(object):
     69      """
     70      Enumeration of build types.
     71      """
     72
     73      SOURCE = 'source'
     74      BINARY = 'binary'
     75
     76      _du = { SOURCE: 'sdist', BINARY: 'bdist' }
     77
     78      @staticmethod
     79 -    def get(key):
     80          try:
     81              return BuildTypes._du[key]
     82          except KeyError:
     83              raise ValueError('Unhandled build type: {0}'.format(key))
     84
     85
     86 -class Console(object):
     87      """
     88      CSB Build Bot. Run with -h for usage.
     89
     90      @param output: build output directory
     91      @type output: str
     92      @param verbosity: verbosity level
     93      @type verbosity: int
     94
     95      @note: The build console automatically detects and builds the csb package
     96             it belongs to. You cannot build a different source tree with it.
     97             See the module documentation for more info.
     98      """
     99
    100      PROGRAM = __file__
    101
    102      USAGE = r"""
    103  CSB Build Console: build, test and package the entire csb project.
    104
    105  Usage:
    106       python {program} -o output [-v verbosity] [-t type] [-h]
    107
    108  Options:
    109        -o  output     Build output directory
    110        -v  verbosity  Verbosity level, default is 1
    111        -t  type       Build type:
    112                          source - build source code distribution (default)
    113                          binary - build executable
    114        -h, --help     Display this help
    115      """
    116
    117 -    def __init__(self, output='.', verbosity=1, buildtype=BuildTypes.SOURCE):
    118
    119          self._input = None
    120          self._output = None
    121          self._temp = None
    122          self._docs = None
    123          self._apidocs = None
    124          self._root = None
    125          self._verbosity = None
    126          self._type = buildtype
    127          self._dist = BuildTypes.get(buildtype)
    128
    129          if os.path.join(SOURCETREE, ROOT) != PARENT:
    130              raise IOError('{0} must be a sub-package or sub-module of {1}'.format(__file__, ROOT))
    131          self._input = SOURCETREE
    132
    133          self._success = True
    134
    135          self.output = output
    136          self.verbosity = verbosity
    137
    138      @property
    139 -    def input(self):
    140          return self._input
    141
    142      @property
    143 -    def output(self):
    144          return self._output
    145      @output.setter
    146 -    def output(self, value):
    147          #value = os.path.dirname(value)
    148          self._output = os.path.abspath(value)
    149          self._temp = os.path.join(self._output, 'build')
    150          self._docs = os.path.join(self._temp, 'docs')
    151          self._apidocs = os.path.join(self._docs, 'api')
    152          self._root = os.path.join(self._temp, ROOT)
    153
    154      @property
    155 -    def verbosity(self):
    156          return self._verbosity
    157      @verbosity.setter
    158 -    def verbosity(self, value):
    159          self._verbosity = int(value)
    160
    161 -    def build(self):
    162          """
    163          Run the console.
    164          """
    165          self.log('\n# Building package {0} from {1}\n'.format(ROOT, SOURCETREE))
    166
    167          self._init()
    168          v = self._revision()
    169          self._doc(v)
    170          self._test()
    171
    172          self._compile()
    173          vn = self._package()
    174
    175          if self._success:
    176              self.log('\n# Done ({0}).\n'.format(vn.full))
    177          else:
    178              self.log('\n# Build failed.\n')
    179
    180
    181 -    def log(self, message, level=1, ending='\n'):
    182
    183          if self._verbosity >= level:
    184              sys.stdout.write(message)
    185              sys.stdout.write(ending)
    186          sys.stdout.flush()
    187
    188 -    def _init(self):
    189          """
    190          Collect all required stuff in the output folder.
    191          """
    192          self.log('# Preparing the file system...')
    193
    194          if not os.path.exists(self._output):
    195              self.log('Creating output directory {0}'.format(self._output), level=2)
    196              os.mkdir(self._output)
    197
    198          if os.path.exists(self._temp):
    199              self.log('Deleting existing temp directory {0}'.format(self._temp), level=2)
    200              shutil.rmtree(self._temp)
    201
    202          self.log('Copying the source tree to temp directory {0}'.format(self._temp), level=2)
    203          shutil.copytree(self._input, self._temp)
    204
    205          if os.path.exists(self._apidocs):
    206              self.log('Deleting existing API docs directory {0}'.format(self._apidocs), level=2)
    207              shutil.rmtree(self._apidocs)
    208          if not os.path.isdir(self._docs):
    209              self.log('Creating docs directory {0}'.format(self._docs), level=2)
    210              os.mkdir(self._docs)
    211          self.log('Creating API docs directory {0}'.format(self._apidocs), level=2)
    212          os.mkdir(self._apidocs)
    213
    214 -    def _revision(self):
    215          """
    216          Write the actual revision number to L{ROOT}.__version__
    217          """
    218          self.log('\n# Setting the most recent Revision Number...')
    219          root = os.path.join(self._root, '__init__.py')
    220
    221          self.log('Retrieving revision number from {0}'.format(root), level=2)
    222          rh = MercurialHandler(root)
    223          revision = rh.read().revision
    224
    225          self.log('Writing back revision number {0}'.format(revision), level=2)
    226          version = rh.write(revision, root)
    227
    228          self.log('  This is {0}.__version__ {1}'.format(ROOT, version), level=1)
    229          csb.__version__ = version
    230
    231          return version
    232
    233 -    def _test(self):
    234          """
    235          Run tests. Also make sure the current environment loads all modules from
    236          the input folder.
    237          """
    238          import csb.test
    239          assert csb.test.__file__.startswith(self._input), 'csb.test not loaded from the input!'     #@UndefinedVariable
    240
    241          from csb.test import unittest
    242
    243          newdata = os.path.join(self._temp, ROOT, 'test', 'data')
    244          csb.test.Config.setDefaultDataRoot(newdata)
    245
    246          self.log('\n# Updating all test pickles in {0} if necessary...'.format(newdata), level=2)
    247          csb.test.Config().ensureDataConsistency()
    248
    249          self.log('\n# Running the Test Console...')
    250
    251          builder = csb.test.AnyTestBuilder()
    252          suite = builder.loadTests(ROOT + '.test.cases.*')
    253
    254          runner = unittest.TextTestRunner(stream=sys.stderr, verbosity=self.verbosity)
    255          result = runner.run(suite)
    256          if result.wasSuccessful():
    257              self.log('\n  Passed all unit tests')
    258          else:
    259              self.log('\n  DID NOT PASS: The build might be broken')
    260
    261 -    def _doc(self, version):
    262          """
    263          Build documentation in the output folder.
    264          """
    265          self.log('\n# Generating API documentation...')
    266          try:
    267              import epydoc.cli
    268          except ImportError:
    269              self.log('\n  Skipped: epydoc is missing')
    270              return
    271
    272          self.log('\n# Emulating ARGV for the Doc Builder...', level=2)
    273          argv = sys.argv
    274          sys.argv = ['epydoc', '--html', '-o', self._apidocs,
    275                      '--name', '{0} v{1}'.format(ROOT.upper(), version),
    276                      '--no-private', '--introspect-only', '--exclude', 'csb.test.cases',
    277                      '--css', os.path.join(self._temp, 'epydoc.css'),
    278                      '--fail-on-error', '--fail-on-warning', '--fail-on-docstring-warning',
    279                      self._root]
    280
    281          if self._verbosity > 0:
    282              sys.argv.append('-v')
    283
    284          try:
    285              epydoc.cli.cli()
    286              sys.exit(0)
    287          except SystemExit as ex:
    288              if ex.code is 0:
    289                  self.log('\n  Passed all doc tests')
    290              else:
    291                  if ex.code == 2:
    292                      self.log('\n  DID NOT PASS: The docs might be broken')
    293                  else:
    294                      self.log('\n  FAIL: Epydoc returned "#{0.code}: {0}"'.format(ex))
    295                      self._success = False
    296
    297          self.log('\n# Restoring the previous ARGV...', level=2)
    298          sys.argv = argv
    299
    300 -    def _compile(self):
    301          """
    302          Byte-compile all modules and packages.
    303          """
    304          self.log('\n# Byte-compiling all *.py files...')
    305
    306          quiet = self.verbosity <= 1
    307          valid = compileall.compile_dir(self._root, quiet=quiet, force=True)
    308
    309          if not valid:
    310              self.log('\n  FAIL: Compilation error(s)\n')
    311              self._success = False
    312
    313 -    def _package(self):
    314          """
    315          Make package.
    316          """
    317          self.log('\n# Configuring CWD and ARGV for the Setup...', level=2)
    318          cwd = os.curdir
    319          os.chdir(self._temp)
    320
    321          if self._verbosity > 1:
    322              verbosity = '-v'
    323          else:
    324              verbosity = '-q'
    325          argv = sys.argv
    326          sys.argv = ['setup.py', verbosity, self._dist, '-d', self._output]
    327
    328          self.log('\n# Building {0} distribution...'.format(self._type))
    329          try:
    330              setup = imp.load_source('setupcsb', 'setup.py')
    331              d = setup.build()
    332              version = setup.VERSION
    333              package = d.dist_files[0][2]
    334
    335              if self._type == BuildTypes.BINARY:
    336                  self._strip_source(package)
    337
    338          except SystemExit as ex:
    339              if ex.code is not 0:
    340                  self.log('\n  FAIL: Setup returned: \n\n{0}\n'.format(ex))
    341                  self._success = False
    342                  package = 'FAIL'
    343
    344          self.log('\n# Restoring the previous CWD and ARGV...', level=2)
    345          os.chdir(cwd)
    346          sys.argv = argv
    347
    348          self.log('  Packaged ' + package)
    349          return version
    350
    351 -    def _strip_source(self, package, source='*.py'):
    352          """
    353          Delete plain text source code files from the package.
    354          """
    355          cwd = os.getcwd()
    356
    357          try:
    358              tmp = os.path.join(self.output, 'tmp')
    359              os.mkdir(tmp)
    360
    361              self.log('\n# Entering {1} in order to delete .py files from {0}...'.format(package, tmp), level=2)
    362              os.chdir(tmp)
    363
    364              oldtar = tarfile.open(package, mode='r:gz')
    365              oldtar.extractall(tmp)
    366              oldtar.close()
    367
    368              newtar = tarfile.open(package, mode='w:gz')
    369
    370              try:
    371                  for i in os.walk('.'):
    372                      for fn in i[2]:
    373                          if fn.endswith('.py'):
    374                              module = os.path.join(i[0], fn);
    375                              if not os.path.isfile(module.replace('.py', '.pyc')):
    376                                  raise ValueError('Missing bytecode for module {0}'.format(module))
    377                              else:
    378                                  os.remove(os.path.join(i[0], fn))
    379
    380                  for i in os.listdir('.'):
    381                      newtar.add(i)
    382              finally:
    383                  newtar.close()
    384
    385          finally:
    386              self.log('\n# Restoring the previous CWD...', level=2)
    387              os.chdir(cwd)
    388              if os.path.exists(tmp):
    389                  shutil.rmtree(tmp)
    390
    391      @staticmethod
    392 -    def exit(message=None, code=0, usage=True):
    393
    394          if message:
    395              print(message)
    396          if usage:
    397              print(Console.USAGE.format(program=Console.PROGRAM))
    398
    399          sys.exit(code)
    400
    401      @staticmethod
    402 -    def run(argv=None):
    403
    404          if argv is None:
    405              argv = sys.argv[1:]
    406
    407          output = None
    408          verb = 1
    409          buildtype = BuildTypes.SOURCE
    410
    411          try:
    412              options, dummy = getopt.getopt(argv, 'o:v:t:h', ['output=', 'verbosity=', 'type=', 'help'])
    413
    414              for option, value in options:
    415                  if option in('-h', '--help'):
    416                      Console.exit(message=None, code=0)
    417                  if option in('-o', '--output'):
    418                      if not os.path.isdir(value):
    419                          Console.exit(message='E: Output directory not found "{0}".'.format(value), code=3)
    420                      output = value
    421                  if option in('-v', '--verbosity'):
    422                      try:
    423                          verb = int(value)
    424                      except ValueError:
    425                          Console.exit(message='E: Verbosity must be an integer.', code=4)
    426                  if option in('-t', '--type'):
    427                      if value not in [BuildTypes.SOURCE, BuildTypes.BINARY]:
    428                          Console.exit(message='E: Invalid build type "{0}".'.format(value), code=5)
    429                      buildtype = value
    430          except getopt.GetoptError as oe:
    431              Console.exit(message='E: ' + str(oe), code=1)
    432
    433          if not output:
    434              Console.exit(code=1, usage=True)
    435          else:
    436              try:
    437                  Console(output, verbosity=verb, buildtype=buildtype).build()
    438              except Exception as ex:
    439                  msg = 'Unexpected Error: {0}\n\n{1}'.format(ex, traceback.format_exc())
    440                  Console.exit(message=msg, code=99, usage=False)
    441
    442
    443 -class RevisionError(RuntimeError):
    444
    445 -    def __init__(self, msg, code, cmd):
    446
    447          super(RevisionError, self).__init__(msg)
    448          self.code = code
    449          self.cmd = cmd
    450
    451 -class RevisionHandler(object):
    452      """
    453      Determines the current repository revision number of a working copy.
    454
    455      @param path: a local checkout path to be examined
    456      @type path: str
    457      @param sc: name of the source control program
    458      @type sc: str
    459      """
    460
    461 -    def __init__(self, path, sc):
    462
    463          self._path = None
    464          self._sc = None
    465
    466          if os.path.exists(path):
    467              self._path = path
    468          else:
    469              raise IOError('Path not found: {0}'.format(path))
    470          if Shell.run([sc, 'help']).code is 0:
    471              self._sc = sc
    472          else:
    473              raise RevisionError('Source control binary probe failed', None, None)
    474
    475      @property
    476 -    def path(self):
    477          return self._path
    478
    479      @property
    480 -    def sc(self):
    481          return self._sc
    482
    483      @abstractmethod
    484 -    def read(self):
    485          """
    486          Return the current revision information.
    487          @rtype: L{RevisionInfo}
    488          """
    489          pass
    490
    491 -    def write(self, revision, sourcefile):
    492          """
    493          Finalize the __version__ = major.minor.micro.{revision} tag.
    494          Overwrite C{sourcefile} in place by substituting the {revision} macro.
    495
    496          @param revision: revision number to write to the source file.
    497          @type revision: int
    498          @param sourcefile: python source file with a __version__ tag, typically
    499                             "csb/__init__.py"
    500          @type sourcefile: str
    501
    502          @return: sourcefile.__version__
    503          """
    504          content = open(sourcefile).readlines()
    505
    506          with open(sourcefile, 'w') as src:
    507              for line in content:
    508                  if line.startswith('__version__'):
    509                      src.write(line.format(revision=revision))
    510                  else:
    511                      src.write(line)
    512
    513          self._delcache(sourcefile)
    514          return imp.load_source('____source', sourcefile).__version__
    515
    516 -    def _run(self, cmd):
    517
    518          si = Shell.run(cmd)
    519          if si.code > 0:
    520              raise RevisionError('SC failed ({0.code}): {0.stderr}'.format(si), si.code, si.cmd)
    521
    522          return si.stdout.splitlines()
    523
    524 -    def _delcache(self, sourcefile):
    525
    526          compiled = os.path.splitext(sourcefile)[0] + '.pyc'
    527          if os.path.isfile(compiled):
    528              os.remove(compiled)
    529
    530          pycache = os.path.join(os.path.dirname(compiled), '__pycache__')
    531          if os.path.isdir(pycache):
    532              shutil.rmtree(pycache)
    533
    534 -class SubversionHandler(RevisionHandler):
    535
    536 -    def __init__(self, path, sc='svn'):
    537          super(SubversionHandler, self).__init__(path, sc)
    538
    539 -    def read(self):
    540
    541          cmd = '{0.sc} info {0.path} -R'.format(self)
    542          maxrevision = None
    543
    544          for line in self._run(cmd):
    545              if line.startswith('Revision:'):
    546                  rev = int(line[9:] .strip())
    547                  if rev > maxrevision:
    548                      maxrevision = rev
    549
    550          if maxrevision is None:
    551              raise RevisionError('No revision number found', code=0, cmd=cmd)
    552
    553          return RevisionInfo(self.path, maxrevision)
    554
    555 -class MercurialHandler(RevisionHandler):
    556
    557 -    def __init__(self, path, sc='hg'):
    558
    559          if os.path.isfile(path):
    560              path = os.path.dirname(path)
    561
    562          super(MercurialHandler, self).__init__(path, sc)
    563
    564 -    def read(self):
    565
    566          wd = os.getcwd()
    567          os.chdir(self.path)
    568
    569          try:
    570              cmd = '{0.sc} log -r tip'.format(self)
    571
    572              revision = None
    573              changeset = ''
    574
    575              for line in self._run(cmd):
    576                  if line.startswith('changeset:'):
    577                      items = line[10:].split(':')
    578                      revision = int(items[0])
    579                      changeset = items[1].strip()
    580                      break
    581
    582              if revision is None:
    583                  raise RevisionError('No revision number found', code=0, cmd=cmd)
    584
    585              return RevisionInfo(self.path, revision, changeset)
    586
    587          finally:
    588              os.chdir(wd)
    589
    590 -class RevisionInfo(object):
    591
    592 -    def __init__(self, item, revision, id=None):
    593
    594          self.item = item
    595          self.revision = revision
    596          self.id = id
    597
    598
    599 -def main():
    600      Console.run()
    601
    602
    603  if __name__ == '__main__':
    604
    605      main()
    606