# Source code for rdflib.plugins.parsers.structureddata

    #!/usr/bin/env python
    """
    Extraction parsers for structured data embedded into HTML or XML files.
    The former may include RDFa or microdata. The syntax and the extraction
    procedures are based on:

    * The RDFa specifications: http://www.w3.org/TR/#tr_RDFa
    * The microdata specification: http://www.w3.org/TR/microdata/
    * The specification of the microdata to RDF conversion:
    http://www.w3.org/TR/microdata-rdf/

    License: W3C Software License,
    http://www.w3.org/Consortium/Legal/copyright-software
    Author: Ivan Herman
    Copyright: W3C

    """

    from rdflib.parser import (
        Parser, StringInputSource, URLInputSource, FileInputSource)

    try:
        import html5lib
        assert html5lib
        html5lib = True
    except ImportError:
        import warnings
        warnings.warn(
            'html5lib not found! RDFa and Microdata ' +
            'parsers will not be available.')
        html5lib = False

    def _get_orig_source(source):
        """
        A bit of a hack; the RDFa/microdata parsers need more than what the
        upper layers of RDFLib provide...
        This method returns the original source references.
        """
        if isinstance(source, StringInputSource):
            orig_source = source.getByteStream()
        elif isinstance(source, URLInputSource):
            orig_source = source.url
        elif isinstance(source, FileInputSource):
            orig_source = source.file.name
            source.file.close()
        else:
            orig_source = source.getByteStream()
        baseURI = source.getPublicId()
        return (baseURI, orig_source)

    def _check_error(graph):
        from .pyRdfa import RDFA_Error, ns_rdf
        from .pyRdfa.options import ns_dc
        for (s, p, o) in graph.triples((None, ns_rdf["type"], RDFA_Error)):
            for (x, y, msg) in graph.triples((s, ns_dc["description"], None)):
                raise Exception("RDFa parsing Error! %s" % msg)

    # This is the parser interface as it would look when called from the
    # rest of RDFLib
    class RDFaParser(Parser):
        """
        Wrapper around the RDFa 1.1 parser. For further details on the RDFa 1.1
        processing, see the relevant W3C documents at
        http://www.w3.org/TR/#tr_RDFa. RDFa 1.1 is defined for XHTML, HTML5, SVG
        and, in general, for any XML language.

        Note that the parser can also handle RDFa 1.0 if the extra parameter is
        used and/or the input source uses RDFa 1.0 specific @version or DTD-s.
        """
        def parse(self, source, graph,
                  pgraph=None,
                  media_type="",
                  rdfa_version=None,
                  embedded_rdf=False,
                  space_preserve=True,
                  vocab_expansion=False,
                  vocab_cache=False,
                  refresh_vocab_cache=False,
                  vocab_cache_report=False,
                  check_lite=False):
            """
            @param source: one of the input sources that the RDFLib package defined
            @type source: InputSource class instance
            @param graph: target graph for the triples; output graph, in RDFa spec.
            parlance
            @type graph: RDFLib Graph
            @keyword pgraph: target for error and warning triples; processor graph,
            in RDFa spec. parlance. If set to None, these triples are ignored
            @type pgraph: RDFLib Graph
            @keyword media_type: explicit setting of the preferred media type
            (a.k.a. content type) of the the RDFa source. None means the content
            type of the HTTP result is used, or a guess is made based on the
            suffix of a file
            @type media_type: string
            @keyword rdfa_version: 1.0 or 1.1. If the value is "", then, by
            default, 1.1 is used unless the source has explicit signals to use
            1.0 (e.g., using a @version attribute, using a DTD set up for 1.0, etc)
            @type rdfa_version: string
            @keyword embedded_rdf: some formats allow embedding RDF in other
            formats: (X)HTML can contain turtle in a special <script> element,
            SVG can have RDF/XML embedded in a <metadata> element. This flag
            controls whether those triples should be interpreted and added to
            the output graph. Some languages (e.g., SVG) require this, and the
            flag is ignored.
            @type embedded_rdf: Boolean
            @keyword space_preserve: by default, space in the HTML source must be preserved in the generated literal;
            this behavior can be switched off
            @type space_preserve: Boolean
            @keyword vocab_expansion: whether the RDFa @vocab attribute should
            also mean vocabulary expansion (see the RDFa 1.1 spec for further
            details)
            @type vocab_expansion: Boolean
            @keyword vocab_cache: in case vocab expansion is used, whether the
            expansion data (i.e., vocabulary) should be cached locally. This
            requires the ability for the local application to write on the
            local file system
            @type vocab_chache: Boolean
            @keyword vocab_cache_report: whether the details of vocabulary file caching process should be reported
            in the processor graph as information (mainly useful for debug)
            @type vocab_cache_report: Boolean
            @keyword refresh_vocab_cache: whether the caching checks of vocabs should be by-passed, ie, if caches should be re-generated regardless of the stored date (important for vocab development)
            @type refresh_vocab_cache: Boolean
            @keyword check_lite: generate extra warnings in case the input source is not RDFa 1.1 check_lite
            @type check_lite: Boolean
            """

            if html5lib is False:
                raise ImportError(
                    'html5lib is not installed, cannot use ' +
                    'RDFa and Microdata parsers.')

            (baseURI, orig_source) = _get_orig_source(source)
            self._process(graph, pgraph, baseURI, orig_source,
                          media_type=media_type,
                          rdfa_version=rdfa_version,
                          embedded_rdf=embedded_rdf,
                          space_preserve=space_preserve,
                          vocab_expansion=vocab_expansion,
                          vocab_cache=vocab_cache,
                          vocab_cache_report=vocab_cache_report,
                          refresh_vocab_cache=refresh_vocab_cache,
                          check_lite=check_lite
                          )

        def _process(self, graph, pgraph, baseURI, orig_source,
                     media_type="",
                     rdfa_version=None,
                     embedded_rdf=False,
                     space_preserve=True,
                     vocab_expansion=False,
                     vocab_cache=False,
                     vocab_cache_report=False,
                     refresh_vocab_cache=False,
                     check_lite=False):
            from .pyRdfa import pyRdfa, Options
            from rdflib import Graph
            processor_graph = pgraph if pgraph is not None else Graph()
            self.options = Options(output_processor_graph=True,
                                   embedded_rdf=embedded_rdf,
                                   space_preserve=space_preserve,
                                   vocab_expansion=vocab_expansion,
                                   vocab_cache=vocab_cache,
                                   vocab_cache_report=vocab_cache_report,
                                   refresh_vocab_cache=refresh_vocab_cache,
                                   check_lite=check_lite)

            if media_type is None:
                media_type = ""
            processor = pyRdfa(self.options,
                               base=baseURI,
                               media_type=media_type,
                               rdfa_version=rdfa_version)
            processor.graph_from_source(orig_source, graph=graph, pgraph=processor_graph, rdfOutput=False)
            # This may result in an exception if the graph parsing led to an error
            _check_error(processor_graph)

    class RDFa10Parser(Parser):
        """
        This is just a convenience class to wrap around the RDFa 1.0 parser.
        """
        def parse(self, source, graph, pgraph=None, media_type=""):
            """
            @param source: one of the input sources that the RDFLib package defined
            @type source: InputSource class instance
            @param graph: target graph for the triples; output graph, in RDFa
            spec. parlance
            @type graph: RDFLib Graph
            @keyword pgraph: target for error and warning triples; processor
            graph, in RDFa spec. parlance. If set to None, these triples are
            ignored
            @type pgraph: RDFLib Graph
            @keyword media_type: explicit setting of the preferred media type
            (a.k.a. content type) of the the RDFa source. None means the content
            type of the HTTP result is used, or a guess is made based on the
            suffix of a file
            @type media_type: string
            @keyword rdfOutput: whether Exceptions should be catched and added,
            as triples, to the processor graph, or whether they should be raised.
            @type rdfOutput: Boolean
            """
            RDFaParser().parse(source, graph, pgraph=pgraph,
                               media_type=media_type, rdfa_version="1.0")

    class MicrodataParser(Parser):
        """
        Wrapper around an HTML5 microdata, extracted and converted into RDF. For
        the specification of microdata, see the relevant section of the HTML5
        spec: http://www.w3.org/TR/microdata/; for the algorithm used to extract
        microdata into RDF, see http://www.w3.org/TR/microdata-rdf/.
        """
        def parse(self, source, graph, vocab_expansion=False, vocab_cache=False):
            """
            @param source: one of the input sources that the RDFLib package defined
            @type source: InputSource class instance
            @param graph: target graph for the triples; output graph, in RDFa
            spec. parlance
            @type graph: RDFLib Graph
            @keyword vocab_expansion: whether the RDFa @vocab attribute should
            also mean vocabulary expansion (see the RDFa 1.1 spec for further
                details)
            @type vocab_expansion: Boolean
            @keyword vocab_cache: in case vocab expansion is used, whether the
            expansion data (i.e., vocabulary) should be cached locally. This
            requires the ability for the local application to write on the
            local file system
            @type vocab_chache: Boolean
            @keyword rdfOutput: whether Exceptions should be catched and added,
            as triples, to the processor graph, or whether they should be raised.
            @type rdfOutput: Boolean
            """
            if html5lib is False:
                raise ImportError(
                    'html5lib is not installed, cannot use RDFa ' +
                    'and Microdata parsers.')

            (baseURI, orig_source) = _get_orig_source(source)
            self._process(graph, baseURI, orig_source,
                          vocab_expansion=vocab_expansion,
                          vocab_cache=vocab_cache)

        def _process(self, graph, baseURI, orig_source,
                     vocab_expansion=False, vocab_cache=False):
            from .pyMicrodata import pyMicrodata
            processor = pyMicrodata(base=baseURI, vocab_expansion=vocab_expansion,
                                    vocab_cache=vocab_cache)
            processor.graph_from_source(
                orig_source, graph=graph, rdfOutput=False)

    class StructuredDataParser(Parser):
        """
        Convenience parser to extract both RDFa (including embedded Turtle)
        and microdata from an HTML file.
        It is simply a wrapper around the specific parsers.
        """
        def parse(self, source, graph,
                  pgraph=None,
                  rdfa_version="",
                  vocab_expansion=False,
                  vocab_cache=False,
                  media_type='text/html'
                  ):
            """
            @param source: one of the input sources that the RDFLib package defined
            @type source: InputSource class instance
            @param graph: target graph for the triples; output graph, in RDFa
            spec. parlance
            @keyword rdfa_version: 1.0 or 1.1. If the value is "", then, by
            default, 1.1 is used unless the source has explicit signals to use 1.0
            (e.g., using a @version attribute, using a DTD set up for 1.0, etc)
            @type rdfa_version: string
            @type graph: RDFLib Graph
            @keyword pgraph: target for error and warning triples; processor
            graph, in RDFa spec. parlance. If set to None, these triples are
            ignored
            @type pgraph: RDFLib Graph
            @keyword vocab_expansion: whether the RDFa @vocab attribute should
            also mean vocabulary expansion (see the RDFa 1.1 spec for further
                details)
            @type vocab_expansion: Boolean
            @keyword vocab_cache: in case vocab expansion is used, whether the
            expansion data (i.e., vocabulary) should be cached locally. This
            requires the ability for the local application to write on the
            local file system
            @type vocab_chache: Boolean
            @keyword rdfOutput: whether Exceptions should be catched and added,
            as triples, to the processor graph, or whether they should be raised.
            @type rdfOutput: Boolean
            """
            # Note that the media_type argument is ignored, and is here only to avoid an 'unexpected argument' error.
            # This parser works for text/html only anyway...
            (baseURI, orig_source) = _get_orig_source(source)
            if rdfa_version == "" : rdfa_version = "1.1"
            RDFaParser()._process(graph, pgraph, baseURI, orig_source,
                                  media_type='text/html',
                                  rdfa_version=rdfa_version,
                                  vocab_expansion=vocab_expansion,
                                  vocab_cache=vocab_cache)
            MicrodataParser()._process(graph, baseURI, orig_source,
                                       vocab_expansion=vocab_expansion,
                                       vocab_cache=vocab_cache)
            from .hturtle import HTurtleParser
            HTurtleParser()._process(graph, baseURI, orig_source, media_type='text/html')