### Navigation

- index
- modules |
- Stetl 1.0.7rc7 documentation »
- Module code »

# Source code for stetl.filters.templatingfilter

    #!/usr/bin/env python
    # -*- coding: utf-8 -*-
    #
    # Transformation of any input using Python Templating as
    # meant in: https://wiki.python.org/moin/Templating.
    # A TemplatingFilter typically is configured with a template file.
    # The input is typically the Template context, the variables to be substituted.
    # The output is a string passed to the next Filter or Output.
    #
    # Author:Just van den Broecke

    from stetl.util import Util, ogr, osr
    from stetl.component import Config
    from stetl.filter import Filter
    from stetl.packet import FORMAT
    from string import Template
    import os

    log = Util.get_log("templatingfilter")

    class TemplatingFilter(Filter):
        """
        Abstract base class for specific template-based filters.
        See https://wiki.python.org/moin/Templating
        Subclasses implement a specific template language like Python string.Template, Mako, Genshi, Jinja2,

        consumes=FORMAT.any, produces=FORMAT.string
        """

        # Start attribute config meta
        # Applying Decorator pattern with the Config class to provide
        # read-only config values from the configured properties.

        @Config(ptype=str, default=None, required=False)
        def template_file(self):
            """
            Path to template file. One of template_file or template_string needs to be configured.
            Required: False
            Default: None
            """
            pass

        @Config(ptype=str, default=None, required=False)
        def template_string(self):
            """
            Template string. One of template_file or template_string needs to be configured.
            Required: False
            Default: None
            """
            pass

        # End attribute config meta

        # Constructor
        def __init__(self, configdict, section, consumes=FORMAT.any, produces=FORMAT.string):
            Filter.__init__(self, configdict, section, consumes, produces)

        def init(self):
            log.info('Init: templating')
            if self.template_file is None and self.template_string is None:
                # If no file present nor template_string error:
                err_s = 'One of template_file or template_string needs to be configured'
                log.error(err_s)
                raise ValueError('One of template_file or template_string needs to be configured')

            self.create_template()

        def exit(self):
            log.info('Exit: templating')
            self.destroy_template()

        def create_template(self):
            '''
            To be overridden in subclasses.
            '''
            pass

        def destroy_template(self):
            pass

        def invoke(self, packet):
            if packet.data is None:
                return packet
            return self.render_template(packet)

        def render_template(self, packet):
            pass

    class StringTemplatingFilter(TemplatingFilter):
        """
        Implements Templating using Python's internal string.Template.
        A template string or file should be configured. The input record
        contains the actual values to be substituted in the template string as a record (key/value pairs).
        Output is a regular string.

        consumes=FORMAT.record, produces=FORMAT.string
        """

        def __init__(self, configdict, section):
            TemplatingFilter.__init__(self, configdict, section, consumes=FORMAT.record)

        def create_template(self):
            # Init once
            if self.template_file is not None:
                # Get template string from file content, otherwise assume template_string is populated
                log.info('Init: reading template file %s ..." % self.template_file')
                template_fp = open(self.template_file, 'r')
                self.template_string = template_fp.read()
                template_fp.close()
                log.info("template file read OK: %s" % self.template_file)

            # Create a standard Python string.Template
            self.template = Template(self.template_string)

        def render_template(self, packet):
            packet.data = self.template.substitute(packet.data)
            return packet

    class Jinja2TemplatingFilter(TemplatingFilter):
        """
        Implements Templating using Jinja2. Jinja2 http://jinja.pocoo.org,
        is a modern and designer-friendly templating language for Python
        modelled after Django’s templates. A 'struct' format as input provides
        a tree-like structure that could originate from a JSON file or REST service.
        This input struct provides all the variables to be inserted into the template.
        The template itself can be configured in this component as a Jinja2 string or -file.
        An optional 'template_search_paths' provides a list of directories from which templates
        can be fethced. Default is the current working directory. Via the optional 'globals_path'
        a JSON structure can be inserted into the Template environment. The variables in this
        globals struture are typically "boilerplate" constants like: id-prefixes, point of contacts etc.

        consumes=FORMAT.struct, produces=FORMAT.string
        """

        # Start attribute config meta
        # Applying Decorator pattern with the Config class to provide
        # read-only config values from the configured properties.

        @Config(ptype=str, default=[os.getcwd()], required=False)
        def template_search_paths(self):
            """
            List of directories where to search for templates, default is current working directory only.
            Required: False
            Default: [os.getcwd()]
            """
            pass

        @Config(ptype=str, default=None, required=False)
        def template_globals_path(self):
            """
            One or more JSON files or URLs with global variables that can be used anywhere in template.
            Multiple files will be merged into one globals dictionary
            Required: False
            Default: None
            """
            pass

        # End attribute config meta

        json_package = None
        ogr_package = ogr
        osr_package = osr

        def __init__(self, configdict, section):
            TemplatingFilter.__init__(self, configdict, section, consumes=[FORMAT.struct, FORMAT.geojson_collection])

        def create_template(self):
            try:
                from jinja2 import Environment, FileSystemLoader
            except Exception, e:
                log.error(
                    'Cannot import modules from Jinja2, err= %s; You probably need to install Jinja2 first, see http://jinja.pocoo.org',
                    str(e))
                raise e

            import json

            Jinja2TemplatingFilter.json_package = json

            # Check for a file with global variables configured in json format
            # TODO get globals in other formats like XML and possibly from a web service
            template_globals = None
            if self.template_globals_path is not None:
                globals_path_list = self.template_globals_path.strip().split(',')

                for file_path in globals_path_list:

                    try:
                        log.info('Read JSON file with globals from: %s', file_path)
                        # Globals can come from local file or remote URL
                        if file_path.startswith('http'):
                            import urllib2

                            fp = urllib2.urlopen(file_path)
                            globals_struct = json.loads(fp.read())
                        else:
                            with open(file_path) as data_file:
                                globals_struct = json.load(data_file)

                        # First file: starts a globals dict, additional globals are merged into that dict
                        if template_globals is None:
                            template_globals = globals_struct
                        else:
                            template_globals.update(globals_struct)

                    except Exception, e:
                        log.error('Cannot read JSON file, err= %s', str(e))
                        raise e

            # Load and Init Template once
            loader = FileSystemLoader(self.template_search_paths)
            self.jinja2_env = Environment(loader=loader, extensions=['jinja2.ext.do'], lstrip_blocks=True, trim_blocks=True)

            # Register additional Filters on the template environment by updating the filters dict:
            self.add_env_filters(self.jinja2_env)

            if self.template_file is not None:
                # Get template string from file content and pass optional globals into context
                self.template = self.jinja2_env.get_template(self.template_file, globals=template_globals)
                log.info("template file read and template created OK: %s" % self.template_file)
            elif self.template_string is None:
                # If no file present, template_string should have been configured
                self.template = self.jinja2_env.from_string(self.template_string, globals=template_globals)

        def render_template(self, packet):
            packet.data = self.template.render(packet.data)
            return packet

        def add_env_filters(self, jinja2_env):
            '''Register additional Filters on the template environment by updating the filters dict:
            Somehow min and max of list are not present so add them as well.
            '''
            jinja2_env.filters['maximum'] = max
            jinja2_env.filters['minimum'] = min
            jinja2_env.filters['geojson2gml'] = Jinja2TemplatingFilter.geojson2gml_filter

        @staticmethod
        def import_ogr():
            if Jinja2TemplatingFilter.ogr_package is None:
                log.error(
                    'Cannot import Python ogr package; You probably need to install GDAL/OGR Python bindings, see https://pypi.python.org/pypi/GDAL')
                raise ImportError

        @staticmethod
        def create_spatial_ref(crs):
            # "crs": {
            #    "type": "EPSG",
            #    "properties": {
            #        "code": "4326"
            #    }
            spatial_ref = Jinja2TemplatingFilter.osr_package.SpatialReference()
            if type(crs) is dict and crs['type'] == 'EPSG':
                # from the GeoJSON source data, though in future deprecated
                spatial_ref.ImportFromEPSG(int(crs['properties']['code']))
            elif type(crs) is str:
                # Like "EPSG:4326"
                spatial_ref.SetFromUserInput(crs)
            elif type(crs) is int:
                # Like 4326
                spatial_ref.ImportFromEPSG(crs)
            return spatial_ref

        @staticmethod
        def geojson2gml_filter(value, source_crs=4326, target_crs=None, gml_id=None, gml_format='GML2', gml_longsrs='NO'):
            """
            Jinja2 custom Filter: generates any GML geometry from a GeoJSON geometry.
            By specifying a target_crs we can even reproject from the source CRS.
            The gml_format=GML2|GML3 determines the general GML form: e.g. pos/posList
            or coordinates. gml_longsrs=YES|NO determines the srsName format like EPSG:4326 or
            urn:ogc:def:crs:EPSG::4326 (long).
            """
            # Import Python OGR lib once
            Jinja2TemplatingFilter.import_ogr()

            try:
                # Create an OGR geometry from a GeoJSON string
                geojson_str = Jinja2TemplatingFilter.json_package.dumps(value)
                geom = Jinja2TemplatingFilter.ogr_package.CreateGeometryFromJson(geojson_str)

                # Create and assign CRS from source CRS
                source_spatial_ref = Jinja2TemplatingFilter.create_spatial_ref(source_crs)
                geom.AssignSpatialReference(source_spatial_ref)

                # Optional: reproject Geometry from source CRS to target CRS
                if target_crs is not None:
                    target_spatial_ref = Jinja2TemplatingFilter.create_spatial_ref(target_crs)
                    transform = Jinja2TemplatingFilter.osr_package.CoordinateTransformation(source_spatial_ref,
                                                                                            target_spatial_ref)
                    geom.Transform(transform)

                # Generate the GML Geometry elements as string, GMLID is optional
                options = ['FORMAT=%s' % gml_format, 'GML3_LONGSRS=%s' % gml_longsrs]
                if gml_id is not None:
                    options.append('GMLID=%s' % gml_id)

                gml_str = geom.ExportToGML(options=options)
            except Exception, e:
                gml_str = 'Failure in CreateGeometryFromJson or ExportToGML, err= %s; check your data and Stetl log' % str(
                    e)
                log.error(gml_str)

            return gml_str

### Navigation

- index
- modules |
- Stetl 1.0.7rc7 documentation »
- Module code »