# Source code for stetl.inputs.fileinput

    # Input classes for ETL, Files.
    #
    # Author: Just van den Broecke
    #
    import csv
    import re
    import fnmatch

    from deprecated.sphinx import deprecated

    from stetl.component import Config
    from stetl.input import Input
    from stetl.util import Util, etree
    from stetl.utils.apachelog import formats, parser
    from stetl.packet import FORMAT

    log = Util.get_log('fileinput')

    class FileInput(Input):
        """
        Abstract base class for specific FileInputs, use derived classes.
        """

        # Start attribute config meta
        # Applying Decorator pattern with the Config class to provide
        # read-only config values from the configured properties.

        @Config(ptype=str, default=None, required=False)
        def file_path(self):
            """
            Path to file or files or URLs: can be a dir or files or URLs
            or even multiple, comma separated. For URLs only JSON is supported now.
            """
            pass

        @Config(ptype=str, default='*.[gxGX][mM][lL]', required=False)
        def filename_pattern(self):
            """
            Filename pattern according to Python ``glob.glob`` for example:
            '\\*.[gxGX][mM][lL]'
            """
            pass

        @Config(ptype=bool, default=False, required=False)
        def depth_search(self):
            """
            Should we recurse into sub-directories to find files?
            """
            pass

        # End attribute config meta

        def __init__(self, configdict, section, produces):
            Input.__init__(self, configdict, section, produces)

            # Create the list of files to be used as input
            self.file_list = Util.make_file_list(self.file_path, None, self.filename_pattern, self.depth_search)
            log.info("file_list=%s" % str(self.file_list))
            if not len(self.file_list):
                raise Exception('File list is empty!!')

            self.cur_file_path = None
            self.file_list_done = []

        def read(self, packet):
            if not len(self.file_list):
                return packet

            file_path = self.file_list.pop(0)

            log.info("Read/parse for start for file=%s...." % file_path)
            packet.data = self.read_file(file_path)
            log.info("Read/parse ok for file=%s" % file_path)

            # One-time read: we're all done
            packet.set_end_of_doc()
            if not len(self.file_list):
                log.info("all files done")
                packet.set_end_of_stream()

            self.file_list_done.append(file_path)
            return packet

        def read_file(self, file_path):
            """
            Override in subclass.
            """
            pass

    class StringFileInput(FileInput):
        """
        Reads and produces file as String.

        produces=FORMAT.string
        """

        # Start attribute config meta
        @Config(ptype=str, default=None, required=False)
        def format_args(self):
            """
            Formatting of content according to Python String.format()
            Input file should have substitutable values like {schema} {foo}
            format_args should be of the form ``format_args = schema:test foo:bar``
            """
            pass

        # End attribute config meta

        def __init__(self, configdict, section):
            FileInput.__init__(self, configdict, section, produces=FORMAT.string)
            self.file = None

            # Optional formatting of content according to Python String.format()
            # Input file should have substitutable values like {schema} {foo}
            # format_args should be of the form format_args = schema:test foo:bar
            if self.format_args:
                # Convert string to dict: http://stackoverflow.com/a/1248990
                self.format_args = Util.string_to_dict(self.format_args, ':')

        def read_file(self, file_path):
            """
            Overridden from base class.
            """

            file_content = None
            with open(file_path, 'r') as f:
                file_content = f.read()
                # Optional: string substitution based on Python String.format()
                # But you can also use StringSubstitutionFilter from filters.
                if self.format_args:
                    file_content = file_content.format(**self.format_args)

            return file_content

    class XmlFileInput(FileInput):
        """
        Parses XML files into etree docs (do not use for large files!).

        produces=FORMAT.etree_doc
        """

        # Constructor
        def __init__(self, configdict, section):
            FileInput.__init__(self, configdict, section, produces=FORMAT.etree_doc)

        def read_file(self, file_path):
            # One-time read/parse only
            data = None
            try:
                data = etree.parse(file_path)
            except Exception as e:
                log.info('file read and parsed NOT OK : %s, err=%s' % (file_path, str(e)))

            return data

    class XmlElementStreamerFileInput(FileInput):
        """
        Extracts XML elements from a file, outputs each feature element in Packet.
        Parsing is streaming (no internal DOM buildup) so any file size can be handled.
        Use this class for your big GML files!

        produces=FORMAT.etree_element
        """

        # Start attribute config meta
        @Config(ptype=list, default=None, required=True)
        def element_tags(self):
            """
            Comma-separated string of XML (feature) element tag names of the elements that should be extracted
            and added to the output element stream.
            """
            pass

        @Config(ptype=bool, default=False, required=False)
        def strip_namespaces(self):
            """
            should namespaces be removed from the input document and thus not be present in the output element stream?
            """
            pass

        # End attribute config meta

        # Constructor
        def __init__(self, configdict, section):
            FileInput.__init__(self, configdict, section, produces=FORMAT.etree_element)
            self.file_list_done = []
            self.context = None
            self.root = None
            self.elem_count = 0
            log.info("Element tags to be matched: %s" % self.element_tags)

        def read(self, packet):
            event = None
            packet.data = None

            if self.context is None:
                if not len(self.file_list):
                    # No more files left, all done
                    log.info("No more files left")
                    return packet

                # Files available: pop next file
                self.cur_file_path = self.file_list.pop(0)
                fd = open(self.cur_file_path, 'rb')
                self.elem_count = 0
                log.info("file opened : %s" % self.cur_file_path)
                self.context = etree.iterparse(fd, events=("start", "end"))
                self.context = iter(self.context)
                event, self.root = next(self.context)

            try:
                event, elem = next(self.context)
            except (etree.XMLSyntaxError, StopIteration):
                # workaround for etree.XMLSyntaxError https://bugs.launchpad.net/lxml/+bug/1185701
                self.context = None

            if self.context is None:
                # Always end of doc
                packet.set_end_of_doc()
                log.info("End of doc: %s elem_count=%d" % (self.cur_file_path, self.elem_count))

                # Maybe end of stream (all docs done)
                if not len(self.file_list):
                    # No more files left: end of stream
                    packet.set_end_of_stream()
                    log.info("End of stream")

                return packet

            # Filter out Namespace from the tag
            # this is the easiest way to go for now
            tag = elem.tag.split('}')
            if len(tag) == 2:
                # Namespaced tag: 2nd is tag
                tag = tag[1]
            else:
                # Non-namespaced tag: first
                tag = tag[0]

            if tag in self.element_tags:
                if event == "start":
                    # TODO check if deepcopy is the right thing to do here.
                    # packet.data = elem
                    pass
                # self.root.remove(elem)
                elif event == "end":
                    # Delete the element from the tree
                    # self.root.clear()
                    packet.data = elem
                    self.elem_count += 1
                    self.root.remove(elem)

                    if self.strip_namespaces:
                        packet.data = Util.stripNamespaces(elem).getroot()

            return packet

    class LineStreamerFileInput(FileInput):
        """
        Reads text-files, producing a stream of lines, one line per Packet.
        NB assumed is that lines in the file have newlines !!
        """

        def __init__(self, configdict, section, produces=FORMAT.line_stream):
            FileInput.__init__(self, configdict, section, produces)
            self.file_list_done = []
            self.file = None

        def read(self, packet):
            # No more files left and done with current file ?
            if not len(self.file_list) and self.file is None:
                packet.set_end_of_stream()
                log.info("EOF file list")
                return packet

            # Done with current file or first file ?
            if self.file is None:
                self.cur_file_path = self.file_list.pop(0)
                self.file = open(self.cur_file_path, 'r')
                log.info("file opened : %s" % self.cur_file_path)

            if packet.is_end_of_stream():
                return packet

            # Assume valid line
            line = self.file.readline()

            # EOF reached ?
            if not line or line == '':
                packet.data = None

                packet.set_end_of_doc()
                log.info("EOF file")
                if self.cur_file_path is not None:
                    self.file_list_done.append(self.cur_file_path)
                    self.cur_file_path = None
                    if not len(self.file_list):
                        # No more files left: end of stream reached
                        packet.set_end_of_stream()
                        log.info("EOF file list")

                self.file = None

                return packet

            try:
                line = line.decode('utf-8')
            except AttributeError:
                # no need to decode
                pass
            packet.data = self.process_line(line)

            return packet

        def process_line(self, line):
            """
            Override in subclass.
            """
            return line

    @deprecated(version='1.0.9', reason='Better to use XmlElementStreamerFileInput for GML features')
    class XmlLineStreamerFileInput(LineStreamerFileInput):
        """
        DEPRECATED Streams lines from an XML file(s)
        NB assumed is that lines in the file have newlines !!

        produces=FORMAT.xml_line_stream
        """

        # Constructor
        def __init__(self, configdict, section):
            LineStreamerFileInput.__init__(self, configdict, section, produces=FORMAT.xml_line_stream)

    class CsvFileInput(FileInput):
        """
        Parse CSV file into stream of records (dict structures) or a one-time record array.
        NB raw version: CSV needs to have first line with fieldnames.

        produces=FORMAT.record or FORMAT.record_array
        """

        @Config(ptype=str, default=',', required=False)
        def delimiter(self):
            """
            A one-character string used to separate fields. It defaults to ','.
            """
            pass

        @Config(ptype=str, default='"', required=False)
        def quote_char(self):
            """
            A one-character string used to quote fields containing special characters, such as the delimiter or quotechar,
            or which contain new-line characters. It defaults to '"'.
            """
            pass

        # Constructor
        def __init__(self, configdict, section):
            FileInput.__init__(self, configdict, section, produces=[FORMAT.record_array, FORMAT.record])
            self.file = None

        def init(self):
            # Init CSV reader
            log.info('Open CSV file: %s', self.file_path)
            self.file = open(self.file_path)

            self.csv_reader = csv.DictReader(self.file, delimiter=self.delimiter, quotechar=self.quote_char)

            if self._output_format == FORMAT.record_array:
                self.arr = list()

        def read(self, packet):
            try:
                # To comply with Stetl record type: force ordinary/base dict-type.
                # Python 3.6+ returns OrderedDict which may not play nice up the Chain
                packet.data = dict(next(self.csv_reader))
                if self._output_format == FORMAT.record_array:
                    while True:
                        self.arr.append(packet.data)
                        packet.data = dict(next(self.csv_reader))

                log.info("CSV row nr %d read: %s" % (self.csv_reader.line_num - 1, packet.data))
            except Exception:
                if self._output_format == FORMAT.record_array:
                    packet.data = self.arr

                packet.set_end_of_stream()
                self.file.close()

            return packet

    class JsonFileInput(FileInput):
        """
        Parse JSON file from file system or URL into hierarchical data struct.
        The struct format may also be a GeoJSON structure. In that case the
        output_format needs to be explicitly set to geojson_collection in the component
        config.

        produces=FORMAT.struct or FORMAT.geojson_collection
        """

        # Constructor
        def __init__(self, configdict, section):
            FileInput.__init__(self, configdict, section, produces=[FORMAT.struct, FORMAT.geojson_collection])

        def read_file(self, file_path):
            # One-time read/parse only
            try:
                import json
                # may read/parse JSON from file or URL
                if file_path.startswith('http'):
                    from urllib.request import urlopen

                    fp = urlopen(file_path)
                    file_data = json.loads(fp.read().decode('utf-8'))
                else:
                    with open(file_path) as data_file:
                        file_data = json.load(data_file)

            except Exception as e:
                log.error('Cannot read JSON from %s, err= %s' % (file_path, str(e)))
                raise e

            return file_data

    class ApacheLogFileInput(FileInput):
        """
        Parses Apache log files. Lines are converted into records based on the log format.
        Log format should follow Apache Log Format. See ApacheLogParser for details.

        produces=FORMAT.record

        """

        @Config(ptype=dict, default={'%l': 'logname', '%>s': 'status', '%D': 'deltat',
                                     '%{User-agent}i': 'agent', '%b': 'bytes', '%{Referer}i': 'referer',
                                     '%u': 'user', '%t': 'time', "'%h": 'host', '%r': 'request'}, required=False)
        def key_map(self):
            """
            Map of cryptic %-field names to readable keys in record.

            """
            pass

        @Config(ptype=str, default=formats['extended'], required=False)
        def log_format(self):
            """
            Log format according to Apache CLF
            """
            pass

        # Constructor
        def __init__(self, configdict, section):
            FileInput.__init__(self, configdict, section, produces=FORMAT.record)
            self.file_list_done = []
            self.file = None
            self.parser = parser(self.log_format, self.key_map, options={'methods': ['GET', 'POST'],
                                                                         'use_native_types': True,
                                                                         'request_path_only': True,
                                                                         'gen_key': True})

        def read(self, packet):
            # No more files left and done with current file ?
            if not len(self.file_list) and self.file is None:
                packet.set_end_of_stream()
                log.info("EOF file list")
                return packet

            # Done with current file or first file ?
            if self.file is None:
                self.cur_file_path = self.file_list.pop(0)
                self.file = open(self.cur_file_path, 'r')
                log.info("file opened : %s" % self.cur_file_path)

            if packet.is_end_of_stream():
                return packet

            # Assume valid line
            line = self.file.readline()

            # EOF reached ?
            if not line or line == '':
                packet.data = None

                packet.set_end_of_doc()
                log.info("EOF file")
                if self.cur_file_path is not None:
                    self.file_list_done.append(self.cur_file_path)
                    self.cur_file_path = None
                    if not len(self.file_list):
                        # No more files left: end of stream reached
                        packet.set_end_of_stream()
                        log.info("EOF file list")

                self.file = None

                return packet

            # Parse logfile line into record (dict)
            packet.data = None
            try:
                packet.data = self.parser.parse(line)
            except Exception as e:
                log.warn("Cannot parse e=%s" % str(e))

            return packet

    class ZipFileInput(FileInput):
        """
        Parse ZIP file from file system or URL into a stream of records containing zipfile-path and file names.

        produces=FORMAT.record
        """

        @Config(ptype=str, default='*', required=False)
        def name_filter(self):
            """
            Regular "glob.glob" expression for filtering out filenames from the ZIP archive.
            """
            pass

        def __init__(self, configdict, section):
            FileInput.__init__(self, configdict, section, produces=FORMAT.record)
            self.file_content = None

            # Pre-compile name filter into regex object to match filenames in zip-archive(s) later
            # See default (*) above, so we alo have a name_filter.
            self.fname_matcher = re.compile(fnmatch.translate(self.name_filter))

        def read(self, packet):
            # No more files left and done with current file ?
            if not self.file_content and not len(self.file_list):
                packet.set_end_of_stream()
                log.info("EOF file list, all files done")
                return packet

            # Done with current file or first file ?
            if self.file_content is None:
                self.cur_file_path = self.file_list.pop(0)

                # Assemble list of file names in archive
                import zipfile
                zf = zipfile.ZipFile(self.cur_file_path, 'r')
                namelist = [{'file_path': self.cur_file_path, 'name': name} for name in zf.namelist()]

                # Apply filename filter to namelist (TODO could be done in single step with previous step)
                self.file_content = [item for item in namelist if self.fname_matcher.match(item["name"])]

                log.info("zip file read : %s filecount=%d" % (self.cur_file_path, len(self.file_content)))

            if len(self.file_content):
                packet.data = self.file_content.pop(0)
                log.info("Pop file record: %s" % str(packet.data))

            if not len(self.file_content):
                self.file_content = None

            return packet

    class GlobFileInput(FileInput):
        """
        Returns file names based on the glob.glob pattern given as filename_filter.

        produces=FORMAT.string or FORMAT.line_stream
        """

        def __init__(self, configdict, section, produces=[FORMAT.string, FORMAT.line_stream]):
            FileInput.__init__(self, configdict, section, produces)

        def read(self, packet):
            if not len(self.file_list):
                return packet

            file_path = self.file_list.pop(0)

            # TODO: os.path.join?
            packet.data = file_path

            # One-time read: we're all done
            packet.set_end_of_doc()
            if not len(self.file_list):
                log.info("all files done")
                packet.set_end_of_stream()

            self.file_list_done.append(file_path)
            return packet