Skip to main content
`Ctrl`+`K`

# Source code for silx.io.configdict

    # /*##########################################################################
    # Copyright (C) 2004-2023 European Synchrotron Radiation Facility
    #
    # This file is part of the PyMca X-ray Fluorescence Toolkit developed at
    # the ESRF by the Software group.
    #
    # Permission is hereby granted, free of charge, to any person obtaining a copy
    # of this software and associated documentation files (the "Software"), to deal
    # in the Software without restriction, including without limitation the rights
    # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    # copies of the Software, and to permit persons to whom the Software is
    # furnished to do so, subject to the following conditions:
    #
    # The above copyright notice and this permission notice shall be included in
    # all copies or substantial portions of the Software.
    #
    # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    # THE SOFTWARE.
    #
    # ############################################################################*/
    """
    This module handles read and write operations to INI files, with data type
    preservation and support for nesting subsections to any depth.

    Data to be written to INI must be stored in a dictionary with string keys.
    Data cannot be stored at the root level of the dictionary, it must be inside
    a sub-dictionary. This means that in the INI file, all parameters must be
    in a section, and if you need a `default` section you must define it
    explicitly.

    Usage example:
    ==============

    Write a dictionary to an INI file::

        from silx.io.configdict import ConfigDict

        ddict = {
                'simple_types': {
                    'float': 1.0,
                    'int': 1,
                    'string': 'Hello World',
                },
                'containers': {
                    'list': [-1, 'string', 3.0, False],
                    'array': numpy.array([1.0, 2.0, 3.0]),
                    'dict': {
                        'key1': 'Hello World',
                        'key2': 2.0,
                    }
                }
            }

        ConfigDict(initdict=ddict).write("foo.ini")

    Read an INI file into a dictionary like structure::

        from silx.io.configdict import ConfigDict

        confdict = ConfigDict()
        confdict.read("foo.ini")

        print("Available sections in INI file:")
        print(confdict.keys())

        for key in confdict:
            for subkey in confdict[key]:
                print("Section %s, parameter %s:" % (key, subkey))
                print(confdict[key][subkey])

    Classes:
    ========

    - :class:`ConfigDict`
    - :class:`OptionStr`

    """

    __author__ = ["E. Papillon", "V.A. Sole", "P. Knobel"]
    __license__ = "MIT"
    __date__ = "15/09/2016"

    import numpy
    import re
    import configparser

    def _boolean(sstr):
        """Coerce a string to a boolean following the same convention as
        :meth:`configparser.ConfigParser.getboolean`:
         - '1', 'yes', 'true' and 'on' cause this function to return ``True``
         - '0', 'no', 'false' and 'off' cause this function to return ``False``

        :param sstr: String representation of a boolean
        :return: ``True`` or ``False``
        :raise: ``ValueError`` if ``sstr`` is not a valid string representation
            of a boolean
        """
        if sstr.lower() in ["1", "yes", "true", "on"]:
            return True
        if sstr.lower() in ["0", "no", "false", "off"]:
            return False
        msg = "Cannot coerce string '%s' to a boolean value. " % sstr
        msg += "Valid boolean strings: '1', 'yes', 'true', 'on',  "
        msg += "'0', 'no', 'false', 'off'"
        raise ValueError(msg)

    def _parse_simple_types(sstr):
        """Coerce a string representation of a value to the most appropriate data
        type, by trial and error.

        Typecasting is attempted to following data types (in this order):
        `int`, `float`, `boolean`. If all of these conversions fail, ``sstr``
        is assumed to be a string.

        :param sstr: String representation of an unknown data type
        :return: Value coerced into the most appropriate data type
        """
        try:
            return int(sstr)
        except ValueError:
            try:
                return float(sstr)
            except ValueError:
                try:
                    return _boolean(sstr)
                except ValueError:
                    if sstr.strip() == "None":
                        return None
                    # un-escape string
                    sstr = sstr.lstrip("\\")
                    # un-escape commas
                    sstr = sstr.replace(r"\,", ",").replace("^@", ",")
                    return sstr

    def _parse_container(sstr):
        """Parse a string representation of a list or a numpy array.

        A string such as ``"-1, Hello World, 3.0"`` is interpreted as the list
        ``[-1, "Hello World", 3.0]``. ``"-1, "no", 3.0\n\t1, 2"`` is interpreted
        a list of 2 lists ``[[-1, False, 3.0], [1, 2]]``

        Strings such as ``"[ [ 1.  2.  3.] [ 4.  5.  6.] ]"`` or
        ``[ 1.0 2.0 3.0 ]`` are interpreted as numpy arrays. Only 1D and 2D
        arrays are permitted.

        :param sstr: String representation of an container type
        :return: List or array
        :raise: ``ValueError`` if string is not a list or an array
        """
        sstr = sstr.strip()

        if not sstr:
            raise ValueError

        if sstr.find(",") == -1:
            # it is not a list
            if (sstr[0] == "[") and (sstr[-1] == "]"):
                # this looks like an array
                try:
                    # try parsing as a 1D array
                    return numpy.array([float(x) for x in sstr[1:-1].split()])
                except ValueError:
                    # try parsing as a 2D array
                    if (sstr[2] == "[") and (sstr[-3] == "]"):
                        nrows = len(sstr[3:-3].split("] ["))
                        data = sstr[3:-3].replace("] [", " ")
                        data = numpy.array([float(x) for x in data.split()])
                        data.shape = nrows, -1
                        return data
            # not a list and not an array
            raise ValueError
        else:
            # if all commas are escaped, it is a strinq, not a list
            if sstr.count(",") == sstr.count(r"\,"):
                raise ValueError

            dataline = [line for line in sstr.splitlines()]
            if len(dataline) == 1:
                return _parse_list_line(dataline[0])
            else:
                return [_parse_list_line(line) for line in dataline]

    def _parse_list_line(sstr):
        """Parse the string representation of a simple 1D list:

        ``"12, 13.1, True, Hello"`` ``->`` ``[12, 13.1, True, "Hello"]``

        :param sstr: String
        :return: List
        """
        sstr = sstr.strip()

        # preserve escaped commas in strings before splitting list
        # (_parse_simple_types recognizes ^@ as a comma)
        sstr.replace(r"\,", "^@")
        # it is a list
        if sstr.endswith(","):
            if "," in sstr[:-1]:
                return [
                    _parse_simple_types(sstr2.strip()) for sstr2 in sstr[:-1].split(",")
                ]
            else:
                return [_parse_simple_types(sstr[:-1].strip())]
        else:
            return [_parse_simple_types(sstr2.strip()) for sstr2 in sstr.split(",")]

    class OptionStr(str):
        """String class providing typecasting methods to parse values in a
        :class:`ConfigDict` generated configuration file.
        """

        def toint(self):
            """
            :return: integer
            :raise: ``ValueError`` if conversion to ``int`` failed
            """
            return int(self)

        def tofloat(self):
            """
            :return: Floating point value
            :raise: ``ValueError`` if conversion to ``float`` failed
            """
            return float(self)

        def toboolean(self):
            """
            '1', 'yes', 'true' and 'on' are interpreted as ``True``

            '0', 'no', 'false' and 'off' are interpreted as ``False``

            :return: Boolean
            :raise: ``ValueError`` if conversion to ``bool`` failed
            """
            return _boolean(self)

        def tostr(self):
            """Return string after replacing escaped commas ``\\,`` with regular
            commas ``,`` and removing leading backslash.

            :return: str(self)
            """
            return str(self.replace(r"\,", ",").lstrip("\\"))

        def tocontainer(self):
            """Return a list or a numpy array.

            Any string containing a comma (``,``) character will be interpreted
            as a list: for instance ``-1, Hello World, 3.0``, or ``2.0,``

            The format for numpy arrays is a blank space delimited list of values
            between square brackets: ``[ 1.3 2.2 3.1 ]``, or
            ``[ [ 1 2 3 ] [ 1 4 9 ] ]``"""
            return _parse_container(self)

        def tobestguess(self):
            """Parse string without prior knowledge of type.

            Conversion to following types is attempted, in this order:
            `list`, `numpy array`, `int`, `float`, `boolean`.
            If all of these conversions fail, the string is returned after
            removing escape characters.
            """
            try:
                return _parse_container(self)
            except ValueError:
                return _parse_simple_types(self)

    class ConfigDict(dict):
        """Store configuration parameters as an ordered dictionary.

        Parameters can be grouped into sections, by storing them as
        sub-dictionaries.

        Keys must be strings. Values can be: integers, booleans, lists,
        numpy arrays, floats, strings.

        Methods are provided to write a configuration file in a variant of INI
        format. A :class:`ConfigDict` can load (or be initialized from) a list of files.

        The main differences between files written/read by this class and standard
        ``ConfigParser`` files are:

            - sections can be nested to any depth
            - value types are guessed when the file is read back
            - to prevent strings from being interpreted as lists, commas are
              escaped with a backslash (``\\,``)
            - strings may be prefixed with a leading backslash (``\\``) to prevent
              conversion to numeric or boolean values

        :param defaultdict: Default dictionary used to initialize the
            :class:`ConfigDict` instance and reinitialize it in case
            :meth:`reset` is called
        :param initdict: Additional initialisation dictionary, added into dict
            after initialisation with ``defaultdict``
        :param filelist: List of configuration files to be read and added into
            dict after ``defaultdict`` and ``initdict``
        """

        def __init__(self, defaultdict=None, initdict=None, filelist=None):
            self.default = defaultdict if defaultdict is not None else {}
            super().__init__(self.default)
            self.filelist = []

            if initdict is not None:
                self.update(initdict)
            if filelist is not None:
                self.read(filelist)

        def reset(self):
            """Revert to default values"""
            self.clear()
            self.update(self.default)

        def clear(self):
            """Clear dictionnary"""
            super().clear()
            self.filelist = []

        def __tolist(self, mylist):
            """If ``mylist` is not a list, encapsulate it in a list and return
            it.

            :param mylist: List to encapsulate
            :returns: ``mylist`` if it is a list, ``[mylist]`` if it isn't
            """
            if mylist is None:
                return None
            if not isinstance(mylist, list):
                return [mylist]
            else:
                return mylist

        def getfiles(self):
            """Return list of configuration file names"""
            return self.filelist

        def getlastfile(self):
            """Return last configuration file name"""
            return self.filelist[len(self.filelist) - 1]

        def __convert(self, option):
            """Used as ``configparser.ConfigParser().optionxform`` to transform
            option names on every read, get, or set operation.

            This overrides the default :mod:`ConfigParser` behavior, in order to
            preserve case rather converting names to lowercase.

            :param option: Option name (any string)
            :return: ``option`` unchanged
            """
            return option

        def read(self, filelist, sections=None):
            """
            Read all specified configuration files into the internal dictionary.

            :param filelist: List of names of files to be added into the internal
                dictionary
            :param sections: If not ``None``, add only the content of the
                specified sections
            :type sections: List
            """
            filelist = self.__tolist(filelist)
            sections = self.__tolist(sections)
            cfg = configparser.ConfigParser()
            cfg.optionxform = self.__convert
            cfg.read(filelist)
            self.__read(cfg, sections)

            for ffile in filelist:
                self.filelist.append([ffile, sections])

        def __read(self, cfg, sections=None):
            """Read a :class:`configparser.ConfigParser` instance into the
            internal dictionary.

            :param cfg: Instance of :class:`configparser.ConfigParser`
            :param sections: If not ``None``, add only the content of the
                specified sections into the internal dictionary
            """
            cfgsect = cfg.sections()

            if sections is None:
                readsect = cfgsect
            else:
                readsect = [sect for sect in cfgsect if sect in sections]

            for sect in readsect:
                ddict = self
                for subsectw in sect.split("."):
                    subsect = subsectw.replace("_|_", ".")
                    if not subsect in ddict:
                        ddict[subsect] = {}
                    ddict = ddict[subsect]
                for opt in cfg.options(sect):
                    ddict[opt] = self.__parse_data(cfg.get(sect, opt))

        def __parse_data(self, data):
            """Parse an option returned by ``ConfigParser``.

            :param data: Option string to be parsed

            The original option is a string, we try to parse it as one of
            following types: `numpx array`, `list`, `float`, `int`, `boolean`,
            `string`
            """
            return OptionStr(data).tobestguess()

        def tostring(self):
            """Return INI file content generated by :meth:`write` as a string"""
            import StringIO

            tmp = StringIO.StringIO()
            self.__write(tmp, self)
            return tmp.getvalue()

        def write(self, ffile):
            """Write the current dictionary to the given filename or
            file handle.

            :param ffile: Output file name or file handle. If a file name is
                provided, the method opens it, writes it and closes it again.
            """
            if not hasattr(ffile, "write"):
                fp = open(ffile, "w")
            else:
                fp = ffile

            self.__write(fp, self)

            if not hasattr(ffile, "write"):
                fp.close()

        def _escape_str(self, sstr):
            """Escape strings and special characters in strings with a ``\\``
            character to ensure they are read back as strings and not parsed.

            :param sstr: String to be escaped
            :returns sstr: String with escape characters (if needed)

            This way, we ensure these strings cannot be interpreted as a numeric
            or boolean types and commas in strings are not interpreted as list
            items separators. We also escape ``%`` when it is not followed by a
            ``(``, as required by :mod:`configparser` because ``%`` is used in
            the interpolation syntax
            (https://docs.python.org/3/library/configparser.html#interpolation-of-values).
            """
            non_str = r"^([0-9]+|[0-9]*\.[0-9]*|none|false|true|on|off|yes|no)$"
            if re.match(non_str, sstr.lower()):
                sstr = "\\" + sstr
            # Escape commas
            sstr = sstr.replace(",", r"\,")

            # Escape % characters except in "%%" and "%("
            sstr = re.sub(r"%([^%\(])", r"%%\1", sstr)

            return sstr

        def __write(self, fp, ddict, secthead=None):
            """Do the actual file writing when called by the ``write`` method.

            :param fp: File handle
            :param ddict: Dictionary to be written to file
            :param secthead: Prefix for section name, used for handling nested
                dictionaries recursively.
            """
            dictkey = []

            for key in ddict.keys():
                if hasattr(ddict[key], "keys"):
                    # subsections are added at the end of a section
                    dictkey.append(key)
                elif isinstance(ddict[key], list):
                    fp.write("%s = " % key)
                    llist = []
                    sep = ", "
                    for item in ddict[key]:
                        if isinstance(item, list):
                            if len(item) == 1:
                                if isinstance(item[0], str):
                                    self._escape_str(item[0])
                                    llist.append("%s," % self._escape_str(item[0]))
                                else:
                                    llist.append("%s," % item[0])
                            else:
                                item2 = []
                                for val in item:
                                    if isinstance(val, str):
                                        val = self._escape_str(val)
                                    item2.append(val)
                                llist.append(", ".join([str(val) for val in item2]))
                            sep = "\n\t"
                        elif isinstance(item, str):
                            llist.append(self._escape_str(item))
                        else:
                            llist.append(str(item))
                    fp.write("%s\n" % (sep.join(llist)))
                elif isinstance(ddict[key], str):
                    fp.write("%s = %s\n" % (key, self._escape_str(ddict[key])))
                else:
                    if isinstance(ddict[key], numpy.ndarray):
                        fp.write(
                            "%s =" % key
                            + " [ "
                            + " ".join([str(val) for val in ddict[key]])
                            + " ]\n"
                        )
                    else:
                        fp.write("%s = %s\n" % (key, ddict[key]))

            for key in dictkey:
                if secthead is None:
                    newsecthead = key.replace(".", "_|_")
                else:
                    newsecthead = "%s.%s" % (secthead, key.replace(".", "_|_"))

                fp.write("\n[%s]\n" % newsecthead)
                self.__write(fp, ddict[key], newsecthead)