Package csb :: Module build

[frames] | no frames]

# Source Code for Module csb.build

      1  """
      2  CSB build related tools and programs.
      3
      4  When executed as a program, this module will run the CSB Build Console and
      5  build the source tree it belongs to. The source tree is added at the
      6  B{beginning} of sys.path to make sure that all subsequent imports from the
      7  Test and Doc consoles will import the right thing (think of multiple CSB
      8  packages installed on the same server).
      9
     10  Here is how to build, test and package the whole project::
     11
     12      $ hg clone https://hg.codeplex.com/csb CSB
     13      $ CSB/csb/build.py -o <output directory>
     14
     15  The Console can also be imported and instantiated as a regular Python class.
     16  In this case the Console again builds the source tree it is part of, but
     17  sys.path will remain intact. Therefore, the Console will assume that all
     18  modules currently in memory, as well as those that can be subsequently imported
     19  by the Console itself, belong to the same CSB package.
     20
     21  @note: The CSB build services no longer support the option to build external
     22         source trees.
     23  @see: [CSB 0000038]
     24  """
     25  from __future__ import print_function
     26
     27  import os
     28  import sys
     29  import getopt
     30  import traceback
     31  import compileall
     32
     33  if os.path.basename(__file__) == '__init__.py':
     34      PARENT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
     35  else:
     36      PARENT = os.path.abspath(os.path.dirname(__file__))
     37
     38  ROOT = 'csb'
     39  SOURCETREE = os.path.abspath(os.path.join(PARENT, ".."))
     40
     41  if __name__ == '__main__':
     42
     43      # make sure "import io" imports the built in module, not csb.io
     44      # io is required by tarfile
     45      for path in sys.path:
     46          if path.startswith(SOURCETREE):
     47              sys.path.remove(path)
     48
     49      import io
     50      assert hasattr(io, 'BufferedIOBase')
     51
     52      sys.path = [SOURCETREE] + sys.path
     53
     54
     55  """
     56  It is now safe to import any modules
     57  """
     58  import imp
     59  import shutil
     60  import tarfile
     61
     62  import csb
     63
     64  from abc import ABCMeta, abstractmethod
     65  from csb.io import Shell
     66
     67
     68 -class BuildTypes(object):
     69      """
     70      Enumeration of build types.
     71      """
     72
     73      SOURCE = 'source'
     74      BINARY = 'binary'
     75
     76      _du = { SOURCE: 'sdist', BINARY: 'bdist' }
     77
     78      @staticmethod
     79 -    def get(key):
     80          try:
     81              return BuildTypes._du[key]
     82          except KeyError:
     83              raise ValueError('Unhandled build type: {0}'.format(key))
     84
     85
     86 -class Console(object):
     87      """
     88      CSB Build Bot. Run with -h for usage.
     89
     90      @param output: build output directory
     91      @type output: str
     92      @param verbosity: verbosity level
     93      @type verbosity: int
     94
     95      @note: The build console automatically detects and builds the csb package
     96             it belongs to. You cannot build a different source tree with it.
     97             See the module documentation for more info.
     98      """
     99
    100      PROGRAM = __file__
    101
    102      USAGE = r"""
    103  CSB Build Console: build, test and package the entire csb project.
    104
    105  Usage:
    106       python {program} -o output [-v verbosity] [-t type] [-h]
    107
    108  Options:
    109        -o  output     Build output directory
    110        -v  verbosity  Verbosity level, default is 1
    111        -t  type       Build type:
    112                          source - build source code distribution (default)
    113                          binary - build executable
    114        -h, --help     Display this help
    115      """
    116
    117 -    def __init__(self, output='.', verbosity=1, buildtype=BuildTypes.SOURCE):
    118
    119          self._input = None
    120          self._output = None
    121          self._temp = None
    122          self._docs = None
    123          self._apidocs = None
    124          self._root = None
    125          self._verbosity = None
    126          self._type = buildtype
    127          self._dist = BuildTypes.get(buildtype)
    128
    129          if os.path.join(SOURCETREE, ROOT) != PARENT:
    130              raise IOError('{0} must be a sub-package or sub-module of {1}'.format(__file__, ROOT))
    131          self._input = SOURCETREE
    132
    133          self._success = True
    134
    135          self.output = output
    136          self.verbosity = verbosity
    137
    138      @property
    139 -    def input(self):
    140          return self._input
    141
    142      @property
    143 -    def output(self):
    144          return self._output
    145      @output.setter
    146 -    def output(self, value):
    147          #value = os.path.dirname(value)
    148          self._output = os.path.abspath(value)
    149          self._temp = os.path.join(self._output, 'build')
    150          self._docs = os.path.join(self._temp, 'docs')
    151          self._apidocs = os.path.join(self._docs, 'api')
    152          self._root = os.path.join(self._temp, ROOT)
    153
    154      @property
    155 -    def verbosity(self):
    156          return self._verbosity
    157      @verbosity.setter
    158 -    def verbosity(self, value):
    159          self._verbosity = int(value)
    160
    161 -    def build(self):
    162          """
    163          Run the console.
    164          """
    165          self.log('\n# Building package {0} from {1}\n'.format(ROOT, SOURCETREE))
    166
    167          self._init()
    168          v = self._revision()
    169          self._doc(v)
    170          self._test()
    171
    172          self._compile()
    173          vn = self._package()
    174
    175          if self._success:
    176              self.log('\n# Done ({0}).\n'.format(vn.full))
    177              return True
    178
    179          self.log('\n# Build failed.\n')
    180          return False
    181
    182 -    def log(self, message, level=1, ending='\n'):
    183
    184          if self._verbosity >= level:
    185              sys.stdout.write(message)
    186              sys.stdout.write(ending)
    187          sys.stdout.flush()
    188
    189 -    def _init(self):
    190          """
    191          Collect all required stuff in the output folder.
    192          """
    193          self.log('# Preparing the file system...')
    194
    195          if not os.path.exists(self._output):
    196              self.log('Creating output directory {0}'.format(self._output), level=2)
    197              os.mkdir(self._output)
    198
    199          if os.path.exists(self._temp):
    200              self.log('Deleting existing temp directory {0}'.format(self._temp), level=2)
    201              shutil.rmtree(self._temp)
    202
    203          self.log('Copying the source tree to temp directory {0}'.format(self._temp), level=2)
    204          shutil.copytree(self._input, self._temp)
    205
    206          if os.path.exists(self._apidocs):
    207              self.log('Deleting existing API docs directory {0}'.format(self._apidocs), level=2)
    208              shutil.rmtree(self._apidocs)
    209          if not os.path.isdir(self._docs):
    210              self.log('Creating docs directory {0}'.format(self._docs), level=2)
    211              os.mkdir(self._docs)
    212          self.log('Creating API docs directory {0}'.format(self._apidocs), level=2)
    213          os.mkdir(self._apidocs)
    214
    215 -    def _revision(self):
    216          """
    217          Write the actual revision number to L{ROOT}.__version__
    218          """
    219          self.log('\n# Setting the most recent Revision Number...')
    220          root = os.path.join(self._root, '__init__.py')
    221
    222          self.log('Retrieving revision number from {0}'.format(root), level=2)
    223          rh = GitHandler(root)
    224          revision = rh.read().revision
    225
    226          self.log('Writing back revision number {0}'.format(revision), level=2)
    227          version = rh.write(revision, root)
    228
    229          self.log('  This is {0}.__version__ {1}'.format(ROOT, version), level=1)
    230          csb.__version__ = version
    231
    232          return version
    233
    234 -    def _test(self):
    235          """
    236          Run tests. Also make sure the current environment loads all modules from
    237          the input folder.
    238          """
    239          import csb.test
    240          assert csb.test.__file__.startswith(self._input), 'csb.test not loaded from the input!'     #@UndefinedVariable
    241
    242          from csb.test import unittest
    243
    244          newdata = os.path.join(self._temp, ROOT, 'test', 'data')
    245          csb.test.Config.setDefaultDataRoot(newdata)
    246          csb.test.Config.setDefaultGeneratedDataRoot(newdata)
    247
    248          self.log('\n# Updating all test pickles in {0} if necessary...'.format(newdata), level=2)
    249          csb.test.Config().ensureDataConsistency()
    250
    251          self.log('\n# Running the Test Console...')
    252
    253          builder = csb.test.AnyTestBuilder()
    254          suite = builder.loadTests(ROOT + '.test.cases.*')
    255
    256          runner = unittest.TextTestRunner(stream=sys.stderr, verbosity=self.verbosity)
    257          result = runner.run(suite)
    258          if result.wasSuccessful():
    259              self.log('\n  Passed all unit tests')
    260          else:
    261              self.log('\n  DID NOT PASS: This build might be broken')
    262              self._success = False
    263
    264 -    def _doc(self, version):
    265          """
    266          Build documentation in the output folder.
    267          """
    268          self.log('\n# Generating API documentation...')
    269          try:
    270              import epydoc.cli
    271          except ImportError:
    272              self.log('\n  Skipped: epydoc is missing')
    273              return
    274
    275          self.log('\n# Emulating ARGV for the Doc Builder...', level=2)
    276          argv = sys.argv
    277          sys.argv = ['epydoc', '--html', '-o', self._apidocs,
    278                      '--name', '{0} v{1}'.format(ROOT.upper(), version),
    279                      '--no-private', '--introspect-only', '--exclude', 'csb.test.cases',
    280                      '--css', os.path.join(self._temp, 'epydoc.css'),
    281                      '--fail-on-error', '--fail-on-warning', '--fail-on-docstring-warning',
    282                      self._root]
    283
    284          if self._verbosity <= 1:
    285              sys.argv.append('-q')
    286
    287          try:
    288              epydoc.cli.cli()
    289              sys.exit(0)
    290          except SystemExit as ex:
    291              if ex.code is 0:
    292                  self.log('\n  Passed all doc tests')
    293              else:
    294                  if ex.code == 2:
    295                      self.log('\n  DID NOT PASS: Generated docs might be broken')
    296                      self._success = False
    297                  else:
    298                      self.log('\n  FAIL: Epydoc returned "#{0.code}: {0}"'.format(ex))
    299                      self._success = False
    300
    301          self.log('\n# Restoring the previous ARGV...', level=2)
    302          sys.argv = argv
    303
    304 -    def _compile(self):
    305          """
    306          Byte-compile all modules and packages.
    307          """
    308          self.log('\n# Byte-compiling all *.py files...')
    309
    310          quiet = self.verbosity <= 1
    311          valid = compileall.compile_dir(self._root, quiet=quiet, force=True)
    312
    313          if not valid:
    314              self.log('\n  FAIL: Compilation error(s)\n')
    315              self._success = False
    316
    317 -    def _package(self):
    318          """
    319          Make package.
    320          """
    321          self.log('\n# Configuring CWD and ARGV for the Setup...', level=2)
    322          cwd = os.curdir
    323          os.chdir(self._temp)
    324
    325          if self._verbosity > 1:
    326              verbosity = '-v'
    327          else:
    328              verbosity = '-q'
    329          argv = sys.argv
    330          sys.argv = ['setup.py', verbosity, self._dist, '-d', self._output]
    331
    332          self.log('\n# Building {0} distribution...'.format(self._type))
    333          version = package = None
    334
    335          try:
    336              setup = imp.load_source('setupcsb', 'setup.py')
    337              d = setup.build()
    338              version = setup.VERSION
    339              package = d.dist_files[0][2]
    340
    341              if self._type == BuildTypes.BINARY:
    342                  self._strip_source(package)
    343
    344          except SystemExit as ex:
    345              if ex.code is not 0:
    346                  self.log('\n  FAIL: Setup returned: \n\n{0}\n'.format(ex))
    347                  self._success = False
    348                  package = 'FAIL'
    349
    350          self.log('\n# Restoring the previous CWD and ARGV...', level=2)
    351          os.chdir(cwd)
    352          sys.argv = argv
    353
    354          self.log('  Packaged ' + package)
    355          return version
    356
    357 -    def _strip_source(self, package, source='*.py'):
    358          """
    359          Delete plain text source code files from the package.
    360          """
    361          cwd = os.getcwd()
    362
    363          try:
    364              tmp = os.path.join(self.output, 'tmp')
    365              os.mkdir(tmp)
    366
    367              self.log('\n# Entering {1} in order to delete .py files from {0}...'.format(package, tmp), level=2)
    368              os.chdir(tmp)
    369
    370              oldtar = tarfile.open(package, mode='r:gz')
    371              oldtar.extractall(tmp)
    372              oldtar.close()
    373
    374              newtar = tarfile.open(package, mode='w:gz')
    375
    376              try:
    377                  for i in os.walk('.'):
    378                      for fn in i[2]:
    379                          if fn.endswith('.py'):
    380                              module = os.path.join(i[0], fn);
    381                              if not os.path.isfile(module.replace('.py', '.pyc')):
    382                                  raise ValueError('Missing bytecode for module {0}'.format(module))
    383                              else:
    384                                  os.remove(os.path.join(i[0], fn))
    385
    386                  for i in os.listdir('.'):
    387                      newtar.add(i)
    388              finally:
    389                  newtar.close()
    390
    391          finally:
    392              self.log('\n# Restoring the previous CWD...', level=2)
    393              os.chdir(cwd)
    394              if os.path.exists(tmp):
    395                  shutil.rmtree(tmp)
    396
    397      @staticmethod
    398 -    def exit(message=None, code=0, usage=True):
    399
    400          if message:
    401              print(message)
    402          if usage:
    403              print(Console.USAGE.format(program=Console.PROGRAM))
    404
    405          sys.exit(code)
    406
    407      @staticmethod
    408 -    def run(argv=None):
    409
    410          if argv is None:
    411              argv = sys.argv[1:]
    412
    413          output = None
    414          verb = 1
    415          buildtype = BuildTypes.SOURCE
    416
    417          try:
    418              options, dummy = getopt.getopt(argv, 'o:v:t:h', ['output=', 'verbosity=', 'type=', 'help'])
    419
    420              for option, value in options:
    421                  if option in('-h', '--help'):
    422                      Console.exit(message=None, code=0)
    423                  if option in('-o', '--output'):
    424                      if not os.path.isdir(value):
    425                          Console.exit(message='E: Output directory not found "{0}".'.format(value), code=3)
    426                      output = value
    427                  if option in('-v', '--verbosity'):
    428                      try:
    429                          verb = int(value)
    430                      except ValueError:
    431                          Console.exit(message='E: Verbosity must be an integer.', code=4)
    432                  if option in('-t', '--type'):
    433                      if value not in [BuildTypes.SOURCE, BuildTypes.BINARY]:
    434                          Console.exit(message='E: Invalid build type "{0}".'.format(value), code=5)
    435                      buildtype = value
    436          except getopt.GetoptError as oe:
    437              Console.exit(message='E: ' + str(oe), code=1)
    438
    439          if not output:
    440              Console.exit(code=1, usage=True)
    441          else:
    442              try:
    443                  ok = Console(output, verbosity=verb, buildtype=buildtype).build()
    444                  Console.exit(code=0 if ok else 66, usage=False)
    445              except Exception as ex:
    446                  msg = 'Unexpected Error: {0}\n\n{1}'.format(ex, traceback.format_exc())
    447                  Console.exit(message=msg, code=99, usage=False)
    448
    449
    450 -class RevisionError(RuntimeError):
    451
    452 -    def __init__(self, msg, code, cmd):
    453
    454          super(RevisionError, self).__init__(msg)
    455          self.code = code
    456          self.cmd = cmd
    457
    458 -class RevisionHandler(object):
    459      """
    460      Determines the current repository revision number of a working copy.
    461
    462      @param path: a local checkout path to be examined
    463      @type path: str
    464      @param sc: name of the source control program
    465      @type sc: str
    466      """
    467
    468 -    def __init__(self, path, sc):
    469
    470          self._path = None
    471          self._sc = None
    472
    473          if os.path.exists(path):
    474              self._path = path
    475          else:
    476              raise IOError('Path not found: {0}'.format(path))
    477          if Shell.run([sc, 'help']).code is 0:
    478              self._sc = sc
    479          else:
    480              raise RevisionError('Source control binary probe failed', None, None)
    481
    482      @property
    483 -    def path(self):
    484          return self._path
    485
    486      @property
    487 -    def sc(self):
    488          return self._sc
    489
    490      @abstractmethod
    491 -    def read(self):
    492          """
    493          Return the current revision information.
    494          @rtype: L{RevisionInfo}
    495          """
    496          pass
    497
    498 -    def write(self, revision, sourcefile):
    499          """
    500          Finalize the __version__ = major.minor.micro.{revision} tag.
    501          Overwrite C{sourcefile} in place by substituting the {revision} macro.
    502
    503          @param revision: revision number to write to the source file.
    504          @type revision: int
    505          @param sourcefile: python source file with a __version__ tag, typically
    506                             "csb/__init__.py"
    507          @type sourcefile: str
    508
    509          @return: sourcefile.__version__
    510          """
    511          content = open(sourcefile).readlines()
    512
    513          with open(sourcefile, 'w') as src:
    514              for line in content:
    515                  if line.startswith('__version__'):
    516                      src.write(line.format(revision=revision))
    517                  else:
    518                      src.write(line)
    519
    520          self._delcache(sourcefile)
    521          return imp.load_source('____source', sourcefile).__version__
    522
    523 -    def _run(self, cmd):
    524
    525          si = Shell.run(cmd)
    526          if si.code > 0:
    527              raise RevisionError('SC failed ({0.code}): {0.stderr}'.format(si), si.code, si.cmd)
    528
    529          return si.stdout.splitlines()
    530
    531 -    def _delcache(self, sourcefile):
    532
    533          compiled = os.path.splitext(sourcefile)[0] + '.pyc'
    534          if os.path.isfile(compiled):
    535              os.remove(compiled)
    536
    537          pycache = os.path.join(os.path.dirname(compiled), '__pycache__')
    538          if os.path.isdir(pycache):
    539              shutil.rmtree(pycache)
    540
    541 -class SubversionHandler(RevisionHandler):
    542
    543 -    def __init__(self, path, sc='svn'):
    544          super(SubversionHandler, self).__init__(path, sc)
    545
    546 -    def read(self):
    547
    548          cmd = '{0.sc} info {0.path} -R'.format(self)
    549          maxrevision = None
    550
    551          for line in self._run(cmd):
    552              if line.startswith('Revision:'):
    553                  rev = int(line[9:] .strip())
    554                  if rev > maxrevision:
    555                      maxrevision = rev
    556
    557          if maxrevision is None:
    558              raise RevisionError('No revision number found', code=0, cmd=cmd)
    559
    560          return RevisionInfo(self.path, maxrevision)
    561
    562 -class MercurialHandler(RevisionHandler):
    563
    564 -    def __init__(self, path, sc='hg'):
    565
    566          if os.path.isfile(path):
    567              path = os.path.dirname(path)
    568
    569          super(MercurialHandler, self).__init__(path, sc)
    570
    571 -    def read(self):
    572
    573          wd = os.getcwd()
    574          os.chdir(self.path)
    575
    576          try:
    577              cmd = '{0.sc} log -r tip'.format(self)
    578
    579              revision = None
    580              changeset = ''
    581
    582              for line in self._run(cmd):
    583                  if line.startswith('changeset:'):
    584                      items = line[10:].split(':')
    585                      revision = int(items[0])
    586                      changeset = items[1].strip()
    587                      break
    588
    589              if revision is None:
    590                  raise RevisionError('No revision number found', code=0, cmd=cmd)
    591
    592              return RevisionInfo(self.path, revision, changeset)
    593
    594          finally:
    595              os.chdir(wd)
    596
    597
    598 -class GitHandler(RevisionHandler):
    599
    600 -    def __init__(self, path, sc='git'):
    601
    602          if os.path.isfile(path):
    603              path = os.path.dirname(path)
    604
    605          super(GitHandler, self).__init__(path, sc)
    606
    607 -    def read(self):
    608
    609          wd = os.getcwd()
    610          os.chdir(self.path)
    611
    612          try:
    613              cmd = '{0.sc} rev-parse --short HEAD'.format(self)
    614              revision = None
    615
    616              for line in self._run(cmd):
    617                  if line.strip():
    618                      revision = line.strip()
    619                      break
    620
    621              if revision is None:
    622                  raise RevisionError('No revision number found', code=0, cmd=cmd)
    623
    624              return RevisionInfo(self.path, revision, revision)
    625
    626          finally:
    627              os.chdir(wd)
    628
    629 -class RevisionInfo(object):
    630
    631 -    def __init__(self, item, revision, id=None):
    632
    633          self.item = item
    634          self.revision = revision
    635          self.id = id
    636
    637
    638 -def main():
    639      Console.run()
    640
    641
    642  if __name__ == '__main__':
    643
    644      main()
    645