Package pygccxml :: Package parser :: Module project_reader

# Source Code for Module pygccxml.parser.project_reader

      1  # Copyright 2004-2008 Roman Yakovenko.
      2  # Distributed under the Boost Software License, Version 1.0. (See
      3  # accompanying file LICENSE_1_0.txt or copy at
      4  # http://www.boost.org/LICENSE_1_0.txt)
      5
      6  import os
      7  import time
      8  import types
      9  import source_reader
     10  import declarations_cache
     11  import pygccxml.declarations
     12  from pygccxml import utils
     13
     14 -class COMPILATION_MODE:
     15      ALL_AT_ONCE = 'all at once'
     16      FILE_BY_FILE = 'file by file'
     17
     18
     19 -class file_configuration_t( object ):
     20      """
     21      file_configuration_t - a class, that contains some data and description how
     22      to treat the data. file_configuration_t can contain reference to the next types
     23      of data:
     24
     25      1) path to C++ source file
     26
     27      2) path to `GCC-XML`_ generated XML file
     28
     29      3) path to C++ source file and path to `GCC-XML`_ generated XML file
     30
     31          In this case, if XML file does not exists, it will be created. Next time
     32          you will ask to parse the source file, the XML file will be used instead.
     33
     34          Small tip: you can setup your makefile to delete XML files every time,
     35          the relevant source file has changed.
     36
     37      4) Python string, that contains valid C++ code
     38
     39
     40      There are few functions, that will help you to construct file_configuration_t
     41      object:
     42
     43      * L{create_source_fc}
     44
     45      * L{create_gccxml_fc}
     46
     47      * L{create_cached_source_fc}
     48
     49      * L{create_text_fc}
     50
     51      """
     52 -    class CONTENT_TYPE:
     53          STANDARD_SOURCE_FILE = 'standard source file'
     54          CACHED_SOURCE_FILE = 'cached source file'
     55          GCCXML_GENERATED_FILE = 'gccxml generated file'
     56          TEXT = 'text'
     57
     58 -    def __init__( self
     59                    , data
     60                    , start_with_declarations=None
     61                    , content_type=CONTENT_TYPE.STANDARD_SOURCE_FILE
     62                    , cached_source_file=None ):
     63          object.__init__( self )
     64          self.__data = data
     65          if not start_with_declarations:
     66              start_with_declarations = []
     67          self.__start_with_declarations = start_with_declarations
     68          self.__content_type = content_type
     69          self.__cached_source_file = cached_source_file
     70          if not self.__cached_source_file \
     71             and self.__content_type == self.CONTENT_TYPE.CACHED_SOURCE_FILE:
     72              self.__cached_source_file = self.__data + '.xml'
     73
     74      @property
     75 -    def data(self):
     76          return self.__data
     77
     78      @property
     79 -    def start_with_declarations(self):
     80          return self.__start_with_declarations
     81
     82      @property
     83 -    def content_type(self):
     84          return self.__content_type
     85
     86      @property
     87 -    def cached_source_file(self):
     88          return self.__cached_source_file
     89
     90 -def create_text_fc( text ):
     91      """
     92      Creates L{file_configuration_t} instance, configured to contain Python string,
     93      that contains valid C++ code
     94
     95      @param text: C++ code
     96      @type text: str
     97
     98      @return: L{file_configuration_t}
     99      """
    100      return file_configuration_t( data=text
    101                                   , content_type=file_configuration_t.CONTENT_TYPE.TEXT )
    102
    103 -def create_source_fc( header ):
    104      """
    105      Creates L{file_configuration_t} instance, configured to contain path to
    106      C++ source file
    107
    108      @param header: path to C++ source file
    109      @type header: str
    110
    111      @return: L{file_configuration_t}
    112      """
    113      return file_configuration_t( data=header
    114                                   , content_type=file_configuration_t.CONTENT_TYPE.STANDARD_SOURCE_FILE )
    115
    116 -def create_gccxml_fc( xml_file ):
    117      """
    118      Creates L{file_configuration_t} instance, configured to contain path to
    119      GCC-XML generated XML file.
    120
    121      @param xml_file: path to GCC-XML generated XML file
    122      @type xml_file: str
    123
    124      @return: L{file_configuration_t}
    125      """
    126      return file_configuration_t( data=xml_file
    127                                   , content_type=file_configuration_t.CONTENT_TYPE.GCCXML_GENERATED_FILE )
    128
    129 -def create_cached_source_fc( header, cached_source_file ):
    130      """
    131      Creates L{file_configuration_t} instance, configured to contain path to
    132      GCC-XML generated XML file and C++ source file. If XML file does not exists,
    133      it will be created and used for parsing. If XML file exists, it will be used
    134      for parsing.
    135
    136      @param header: path to C++ source file
    137      @type header: str
    138
    139      @param cached_source_file: path to GCC-XML generated XML file
    140      @type cached_source_file: str
    141
    142      @return: L{file_configuration_t}
    143      """
    144      return file_configuration_t( data=header
    145                                   , cached_source_file=cached_source_file
    146                                   , content_type=file_configuration_t.CONTENT_TYPE.CACHED_SOURCE_FILE )
    147
    148 -class project_reader_t:
    149      """Parses header files and returns the contained declarations.
    150      """
    151 -    def __init__( self, config, cache=None, decl_factory=None):
    152          """Constructor.
    153
    154          config is a configuration object that contains the parameters
    155          for invoking gccxml. cache specifies the cache to use for
    156          caching declarations between separate runs. By default, no
    157          cache is used.  decl_factory is an object that must provide
    158          the same interface than
    159          L{decl_factory_t<declarations.decl_factory_t>}, i.e. there must
    160          be a set of C{create_*} methods that return an instance of an
    161          appropriate declaration class.  By default, the declaration
    162          classes defined in the L{declarations} package are used.
    163
    164          @param config: Configuration object
    165          @type config: L{config_t}
    166          @param cache: Declaration cache (None=no cache)
    167          @type cache: L{cache_base_t} or str
    168          @param decl_factory: Custom declaration factory object or None
    169          @type decl_factory: decl_factory_t
    170          """
    171          self.__config = config
    172          self.__dcache = None
    173          if isinstance( cache, declarations_cache.cache_base_t ):
    174              self.__dcache = cache
    175          elif isinstance( cache, types.StringTypes ):
    176              self.__dcache = declarations_cache.file_cache_t(cache)
    177          else:
    178              self.__dcache = declarations_cache.dummy_cache_t()
    179          self.__decl_factory = decl_factory
    180          if not decl_factory:
    181              self.__decl_factory = pygccxml.declarations.decl_factory_t()
    182
    183          self.logger = utils.loggers.gccxml
    184
    185      @staticmethod
    186 -    def get_os_file_names( files ):
    187          """Returns a list of OS file names
    188
    189          @param files: list of strings or L{file_configuration_t} instances.
    190                        files could contain a mix of them
    191          @type files: list
    192          """
    193          fnames = []
    194          for f in files:
    195              if isinstance( f, types.StringTypes ):
    196                  fnames.append( f )
    197              elif isinstance( f, file_configuration_t ):
    198                  if f.content_type in ( file_configuration_t.CONTENT_TYPE.STANDARD_SOURCE_FILE
    199                                         , file_configuration_t.CONTENT_TYPE.CACHED_SOURCE_FILE ):
    200                      fnames.append( f.data )
    201              else:
    202                  pass
    203          return fnames
    204
    205 -    def read_files( self, files, compilation_mode=COMPILATION_MODE.FILE_BY_FILE):
    206          """Parse header files.
    207
    208          @param files: list of strings or L{file_configuration_t} instances.
    209                        files could contain a mix of them
    210          @type files: list
    211          @param compilation_mode: Determines whether the files are parsed individually or as one single chunk
    212          @type compilation_mode: L{COMPILATION_MODE}
    213          @returns: Declarations
    214          """
    215          if compilation_mode == COMPILATION_MODE.ALL_AT_ONCE \
    216             and len( files ) == len( self.get_os_file_names(files) ):
    217              return self.__parse_all_at_once(files)
    218          else:
    219              if compilation_mode == COMPILATION_MODE.ALL_AT_ONCE:
    220                  msg = ''.join([
    221                      "Unable to parse files using ALL_AT_ONCE mode. "
    222                      , "There is some file configuration that is not file. "
    223                      , "pygccxml.parser.project_reader_t switches to FILE_BY_FILE mode." ])
    224                  self.logger.warning( msg )
    225              return self.__parse_file_by_file(files)
    226
    227 -    def __parse_file_by_file(self, files):
    228          namespaces = []
    229          config = self.__config.clone()
    230          self.logger.debug( "Reading project files: file by file" )
    231          for prj_file in files:
    232              reader = None
    233              header = None
    234              content_type = None
    235              if isinstance( prj_file, file_configuration_t ):
    236                  del config.start_with_declarations[:]
    237                  config.start_with_declarations.extend( prj_file.start_with_declarations )
    238                  header = prj_file.data
    239                  content_type = prj_file.content_type
    240              else:
    241                  config = self.__config
    242                  header = prj_file
    243                  content_type = file_configuration_t.CONTENT_TYPE.STANDARD_SOURCE_FILE
    244              reader = source_reader.source_reader_t( config
    245                                                      , self.__dcache
    246                                                      , self.__decl_factory )
    247              decls = None
    248              if content_type == file_configuration_t.CONTENT_TYPE.STANDARD_SOURCE_FILE:
    249                  self.logger.info( 'Parsing source file "%s" ... ' % header )
    250                  decls = reader.read_file( header )
    251              elif content_type == file_configuration_t.CONTENT_TYPE.GCCXML_GENERATED_FILE:
    252                  self.logger.info( 'Parsing xml file "%s" ... ' % header )
    253                  decls = reader.read_xml_file( header )
    254              elif content_type == file_configuration_t.CONTENT_TYPE.CACHED_SOURCE_FILE:
    255                  #TODO: raise error when header file does not exist
    256                  if not os.path.exists( prj_file.cached_source_file ):
    257                      dir_ = os.path.split( prj_file.cached_source_file )[0]
    258                      if dir_ and not os.path.exists( dir_ ):
    259                          os.makedirs( dir_ )
    260                      self.logger.info( 'Creating xml file "%s" from source file "%s" ... '
    261                                        % ( prj_file.cached_source_file, header ) )
    262                      reader.create_xml_file( header, prj_file.cached_source_file )
    263                  self.logger.info( 'Parsing xml file "%s" ... ' % prj_file.cached_source_file )
    264                  decls = reader.read_xml_file( prj_file.cached_source_file )
    265              else:
    266                  decls = reader.read_string( header )
    267              namespaces.append( decls )
    268          self.logger.debug( "Flushing cache... " )
    269          start_time = time.clock()
    270          self.__dcache.flush()
    271          self.logger.debug( "Cache has been flushed in %.1f secs" % ( time.clock() - start_time ) )
    272          answer = []
    273          self.logger.debug( "Joining namespaces ..." )
    274          for file_nss in namespaces:
    275              answer = self._join_top_namespaces( answer, file_nss )
    276          self.logger.debug( "Joining declarations ..." )
    277          for ns in answer:
    278              if isinstance( ns, pygccxml.declarations.namespace_t ):
    279                  self._join_declarations( ns )
    280          leaved_classes = self._join_class_hierarchy( answer )
    281          types = self.__declarated_types(answer)
    282          self.logger.debug( "Relinking declared types ..." )
    283          self._relink_declarated_types( leaved_classes, types )
    284          source_reader.bind_aliases( pygccxml.declarations.make_flatten( answer ) )
    285          return answer
    286
    287 -    def __parse_all_at_once(self, files):
    288          config = self.__config.clone()
    289          self.logger.debug( "Reading project files: all at once" )
    290          header_content = []
    291          for header in files:
    292              if isinstance( header, file_configuration_t ):
    293                  del config.start_with_declarations[:]
    294                  config.start_with_declarations.extend( header.start_with_declarations )
    295                  header_content.append( '#include "%s" %s' % ( header.data, os.linesep ) )
    296              else:
    297                  header_content.append( '#include "%s" %s' % ( header, os.linesep ) )
    298          return self.read_string( ''.join( header_content ) )
    299
    300 -    def read_string(self, content):
    301          """Parse a string containing C/C++ source code.
    302
    303          @param content: C/C++ source code.
    304          @type content: str
    305          @returns: Declarations
    306          """
    307          reader = source_reader.source_reader_t( self.__config, None, self.__decl_factory )
    308          return reader.read_string( content )
    309
    310 -    def _join_top_namespaces(self, main_ns_list, other_ns_list):
    311          answer = main_ns_list[:]
    312          for other_ns in other_ns_list:
    313              main_ns = pygccxml.declarations.find_declaration( answer
    314                                                                , type=pygccxml.declarations.namespace_t
    315                                                                , name=other_ns._name
    316                                                                , recursive=False )
    317              if main_ns:
    318                  main_ns.take_parenting( other_ns )
    319              else:
    320                  answer.append( other_ns )
    321          return answer
    322
    323 -    def _join_namespaces( self, nsref ):
    324          assert isinstance( nsref, pygccxml.declarations.namespace_t )
    325          ddhash = {} # decl.__class__ :  { decl.name : [decls] } double declaration hash
    326          decls = []
    327
    328          for decl in nsref.declarations:
    329              if not ddhash.has_key( decl.__class__ ):
    330                  ddhash[ decl.__class__ ] = { decl._name : [ decl ] }
    331                  decls.append( decl )
    332              else:
    333                  joined_decls = ddhash[ decl.__class__ ]
    334                  if not joined_decls.has_key( decl._name ):
    335                      decls.append( decl )
    336                      joined_decls[decl._name] = [ decl ]
    337                  else:
    338                      if isinstance( decl, pygccxml.declarations.calldef_t ):
    339                          if decl not in joined_decls[decl._name]:
    340                              #functions has overloading
    341                              decls.append( decl )
    342                              joined_decls[decl._name].append( decl )
    343                      elif isinstance( decl, pygccxml.declarations.enumeration_t ):
    344                          #unnamed enums
    345                          if not decl.name and decl not in joined_decls[decl._name]:
    346                              decls.append( decl )
    347                              joined_decls[decl._name].append( decl )
    348                      elif isinstance( decl, pygccxml.declarations.class_t ):
    349                          #unnamed classes
    350                          if not decl.name and decl not in joined_decls[decl._name]:
    351                              decls.append( decl )
    352                              joined_decls[decl._name].append( decl )
    353                      else:
    354                          assert 1 == len( joined_decls[ decl._name ] )
    355                          if isinstance( decl, pygccxml.declarations.namespace_t ):
    356                              joined_decls[ decl._name ][0].take_parenting( decl )
    357          nsref.declarations = decls
    358
    359 -    def _join_class_hierarchy( self, namespaces ):
    360          create_key = lambda decl:( decl.location.as_tuple()
    361                                     , tuple( pygccxml.declarations.declaration_path( decl ) ) )
    362          classes = filter( lambda decl: isinstance(decl, pygccxml.declarations.class_t )
    363                            , pygccxml.declarations.make_flatten( namespaces ) )
    364          leaved_classes = {}
    365          #selecting classes to leave
    366          for class_ in classes:
    367              key = create_key( class_ )
    368              if key not in leaved_classes:
    369                  leaved_classes[ key ] = class_
    370          #replacing base and derived classes with those that should be leave
    371          #also this loop will add missing derived classes to the base
    372          for class_ in classes:
    373              leaved_class = leaved_classes[create_key( class_ )]
    374              for base_info in class_.bases:
    375                  leaved_base = leaved_classes[ create_key( base_info.related_class ) ]
    376                  #treating base class hierarchy of leaved_class
    377                  leaved_base_info = pygccxml.declarations.hierarchy_info_t(
    378                      related_class=leaved_base
    379                      , access=base_info.access )
    380                  if leaved_base_info not in leaved_class.bases:
    381                      leaved_class.bases.append( leaved_base_info )
    382                  else:
    383                      index = leaved_class.bases.index( leaved_base_info )
    384                      leaved_class.bases[index].related_class = leaved_base_info.related_class
    385                  #treating derived class hierarchy of leaved_base
    386                  leaved_derived_for_base_info = pygccxml.declarations.hierarchy_info_t(
    387                      related_class=leaved_class
    388                      , access=base_info.access )
    389                  if leaved_derived_for_base_info not in leaved_base.derived:
    390                      leaved_base.derived.append( leaved_derived_for_base_info )
    391                  else:
    392                      index = leaved_base.derived.index( leaved_derived_for_base_info )
    393                      leaved_base.derived[index].related_class = leaved_derived_for_base_info.related_class
    394              for derived_info in class_.derived:
    395                  leaved_derived = leaved_classes[ create_key( derived_info.related_class ) ]
    396                  #treating derived class hierarchy of leaved_class
    397                  leaved_derived_info = pygccxml.declarations.hierarchy_info_t(
    398                      related_class=leaved_derived
    399                      , access=derived_info.access )
    400                  if leaved_derived_info not in leaved_class.derived:
    401                      leaved_class.derived.append( leaved_derived_info )
    402                  #treating base class hierarchy of leaved_derived
    403                  leaved_base_for_derived_info = pygccxml.declarations.hierarchy_info_t(
    404                      related_class=leaved_class
    405                      , access=derived_info.access )
    406                  if leaved_base_for_derived_info not in leaved_derived.bases:
    407                      leaved_derived.bases.append( leaved_base_for_derived_info )
    408          #this loops remove instance we from parent.declarations
    409          for class_ in classes:
    410              key = create_key( class_ )
    411              if id( leaved_classes[key] ) == id( class_ ):
    412                  continue
    413              else:
    414                  declarations = None
    415                  if class_.parent:
    416                      declarations = class_.parent.declarations
    417                  else:
    418                      declarations = namespaces #yes, we are talking about global class that doesn't
    419                      #belong to any namespace. Usually is compiler generated top level classes
    420                  declarations_ids = [ id(decl) for decl in declarations ]
    421                  del declarations[ declarations_ids.index( id(class_) ) ]
    422          return leaved_classes
    423
    424 -    def _relink_declarated_types(self, leaved_classes, declarated_types):
    425          create_key = lambda decl:( decl.location.as_tuple()
    426                                     , tuple( pygccxml.declarations.declaration_path( decl ) ) )
    427          for decl_wrapper_type in declarated_types:
    428              #it is possible, that cache contains reference to dropped class
    429              #We need to clear it
    430              decl_wrapper_type.cache.reset()
    431              if isinstance( decl_wrapper_type.declaration, pygccxml.declarations.class_t ):
    432                  key = create_key(decl_wrapper_type.declaration)
    433                  if leaved_classes.has_key( key ):
    434                      decl_wrapper_type.declaration = leaved_classes[ create_key(decl_wrapper_type.declaration) ]
    435                  else:
    436                      if decl_wrapper_type.declaration._name.startswith( '__vmi_class_type_info_pseudo' ):
    437                          continue
    438                      msg = []
    439                      msg.append( "Unable to find out actual class definition: '%s'." % decl_wrapper_type.declaration._name )
    440                      msg.append( "Class definition has been changed from one compilation to an other." )
    441                      msg.append( "Why did it happen to me? Here is a short list of reasons: " )
    442                      msg.append( "    1. There are different preprocessor definitions applied on same file during compilation" )
    443                      msg.append( "    2. Bug in pygccxml." )
    444                      self.logger.error( os.linesep.join(msg) )
    445
    446 -    def _join_declarations( self, declref ):
    447          self._join_namespaces( declref )
    448          for ns in declref.declarations:
    449              if isinstance( ns, pygccxml.declarations.namespace_t ):
    450                  self._join_declarations( ns )
    451
    452 -    def __declarated_types(self, namespaces):
    453          def get_from_type(cpptype):
    454              if not cpptype:
    455                  return []
    456              elif isinstance( cpptype, pygccxml.declarations.fundamental_t ):
    457                  return []
    458              elif isinstance( cpptype, pygccxml.declarations.declarated_t ):
    459                  return [ cpptype ]
    460              elif isinstance( cpptype, pygccxml.declarations.compound_t ):
    461                  return get_from_type( cpptype.base )
    462              elif isinstance( cpptype, pygccxml.declarations.calldef_type_t ):
    463                  types = get_from_type( cpptype.return_type )
    464                  for arg in cpptype.arguments_types:
    465                      types.extend( get_from_type( arg ) )
    466                  return types
    467              else:
    468                  assert isinstance( cpptype, ( pygccxml.declarations.unknown_t
    469                                                , pygccxml.declarations.ellipsis_t ) )
    470                  return []
    471          types = []
    472          for decl in pygccxml.declarations.make_flatten( namespaces ):
    473              if isinstance( decl, pygccxml.declarations.calldef_t ):
    474                  types.extend( get_from_type( decl.function_type() ) )
    475              elif isinstance( decl, (pygccxml.declarations.typedef_t, pygccxml.declarations.variable_t) ):
    476                  types.extend( get_from_type( decl.type ) )
    477          return types
    478