Hide navigation sidebar
Hide table of contents sidebar
Back to top
Toggle table of contents sidebar

# Source code for gevent._config

    # Copyright (c) 2018 gevent. See LICENSE for details.
    """
    gevent tunables.

    This should be used as ``from gevent import config``. That variable
    is an object of :class:`Config`.

    .. versionadded:: 1.3a2

    .. versionchanged:: 22.08.0
       Invoking this module like ``python -m gevent._config`` will
       print a help message about available configuration properties.
       This is handy to quickly look for environment variables.
    """

    from __future__ import print_function, absolute_import, division

    import importlib
    import os
    import textwrap

    from gevent._compat import string_types
    from gevent._compat import WIN

    __all__ = [
        'config',
    ]

    ALL_SETTINGS = []

    class SettingType(type):
        # pylint:disable=bad-mcs-classmethod-argument

        def __new__(cls, name, bases, cls_dict):
            if name == 'Setting':
                return type.__new__(cls, name, bases, cls_dict)

            cls_dict["order"] = len(ALL_SETTINGS)
            if 'name' not in cls_dict:
                cls_dict['name'] = name.lower()

            if 'environment_key' not in cls_dict:
                cls_dict['environment_key'] = 'GEVENT_' + cls_dict['name'].upper()

            new_class = type.__new__(cls, name, bases, cls_dict)
            new_class.fmt_desc(cls_dict.get("desc", ""))
            new_class.__doc__ = new_class.desc
            ALL_SETTINGS.append(new_class)

            if new_class.document:
                setting_name = cls_dict['name']

                def getter(self):
                    return self.settings[setting_name].get()

                def setter(self, value): # pragma: no cover
                    # The setter should never be hit, Config has a
                    # __setattr__ that would override. But for the sake
                    # of consistency we provide one.
                    self.settings[setting_name].set(value)

                prop = property(getter, setter, doc=new_class.__doc__)

                setattr(Config, cls_dict['name'], prop)
            return new_class

        def fmt_desc(cls, desc):
            desc = textwrap.dedent(desc).strip()
            if hasattr(cls, 'shortname_map'):
                desc += (
                    "\n\nThis is an importable value. It can be "
                    "given as a string naming an importable object, "
                    "or a list of strings in preference order and the first "
                    "successfully importable object will be used. (Separate values "
                    "in the environment variable with commas.) "
                    "It can also be given as the callable object itself (in code). "
                )
                if cls.shortname_map:
                    desc += "Shorthand names for default objects are %r" % (list(cls.shortname_map),)
            if getattr(cls.validate, '__doc__'):
                desc += '\n\n' + textwrap.dedent(cls.validate.__doc__).strip()
            if isinstance(cls.default, str) and hasattr(cls, 'shortname_map'):
                default = "`%s`" % (cls.default,)
            else:
                default = "`%r`" % (cls.default,)
            desc += "\n\nThe default value is %s" % (default,)
            desc += ("\n\nThe environment variable ``%s`` "
                     "can be used to control this." % (cls.environment_key,))
            setattr(cls, "desc", desc)
            return desc

    def validate_invalid(value):
        raise ValueError("Not a valid value: %r" % (value,))

    def validate_bool(value):
        """
        This is a boolean value.

        In the environment variable, it may be given as ``1``, ``true``,
        ``on`` or ``yes`` for `True`, or ``0``, ``false``, ``off``, or
        ``no`` for `False`.
        """
        if isinstance(value, string_types):
            value = value.lower().strip()
            if value in ('1', 'true', 'on', 'yes'):
                value = True
            elif value in ('0', 'false', 'off', 'no') or not value:
                value = False
            else:
                raise ValueError("Invalid boolean string: %r" % (value,))
        return bool(value)

    def validate_anything(value):
        return value

    convert_str_value_as_is = validate_anything

    class Setting(object):
        name = None
        value = None
        validate = staticmethod(validate_invalid)
        default = None
        environment_key = None
        document = True

        desc = """\

        A long ReST description.

        The first line should be a single sentence.

        """

        def _convert(self, value):
            if isinstance(value, string_types):
                return value.split(',')
            return value

        def _default(self):
            result = os.environ.get(self.environment_key, self.default)
            result = self._convert(result)
            return result

        def get(self):
            # If we've been specifically set, return it
            if 'value' in self.__dict__:
                return self.value
            # Otherwise, read from the environment and reify
            # so we return consistent results.
            self.value = self.validate(self._default())
            return self.value

        def set(self, val):
            self.value = self.validate(self._convert(val))

    Setting = SettingType('Setting', (Setting,), dict(Setting.__dict__))

    def make_settings():
        """
        Return fresh instances of all classes defined in `ALL_SETTINGS`.
        """
        settings = {}
        for setting_kind in ALL_SETTINGS:
            setting = setting_kind()
            assert setting.name not in settings
            settings[setting.name] = setting
        return settings

    class Config(object):
        """
        Global configuration for gevent.

        There is one instance of this object at ``gevent.config``. If you
        are going to make changes in code, instead of using the documented
        environment variables, you need to make the changes before using
        any parts of gevent that might need those settings. For example::

            >>> from gevent import config
            >>> config.fileobject = 'thread'

            >>> from gevent import fileobject
            >>> fileobject.FileObject.__name__
            'FileObjectThread'

        .. versionadded:: 1.3a2

        """

        def __init__(self):
            self.settings = make_settings()

        def __getattr__(self, name):
            if name not in self.settings:
                raise AttributeError("No configuration setting for: %r" % name)
            return self.settings[name].get()

        def __setattr__(self, name, value):
            if name != "settings" and name in self.settings:
                self.set(name, value)
            else:
                super(Config, self).__setattr__(name, value)

        def set(self, name, value):
            if name not in self.settings:
                raise AttributeError("No configuration setting for: %r" % name)
            self.settings[name].set(value)

        def __dir__(self):
            return list(self.settings)

        def print_help(self):
            for k, v in self.settings.items():
                print(k)
                print(textwrap.indent(v.__doc__.lstrip(), ' ' * 4))
                print()

    class ImportableSetting(object):

        def _import_one_of(self, candidates):
            assert isinstance(candidates, list)
            if not candidates:
                raise ImportError('Cannot import from empty list')

            for item in candidates[:-1]:
                try:
                    return self._import_one(item)
                except ImportError:
                    pass

            return self._import_one(candidates[-1])

        def _import_one(self, path, _MISSING=object()):
            if not isinstance(path, string_types):
                return path

            if '.' not in path or '/' in path:
                raise ImportError("Cannot import %r. "
                                  "Required format: [package.]module.class. "
                                  "Or choose from %r"
                                  % (path, list(self.shortname_map)))

            module, item = path.rsplit('.', 1)
            module = importlib.import_module(module)
            x = getattr(module, item, _MISSING)
            if x is _MISSING:
                raise ImportError('Cannot import %r from %r' % (item, module))
            return x

        shortname_map = {}

        def validate(self, value):
            if isinstance(value, type):
                return value
            return self._import_one_of([self.shortname_map.get(x, x) for x in value])

        def get_options(self):
            result = {}
            for name, val in self.shortname_map.items():
                try:
                    result[name] = self._import_one(val)
                except ImportError as e:
                    result[name] = e
            return result

    class BoolSettingMixin(object):
        validate = staticmethod(validate_bool)
        # Don't do string-to-list conversion.
        _convert = staticmethod(convert_str_value_as_is)

    class IntSettingMixin(object):
        # Don't do string-to-list conversion.
        def _convert(self, value):
            if value:
                return int(value)

        validate = staticmethod(validate_anything)

    class _PositiveValueMixin(object):

        def validate(self, value):
            if value is not None and value <= 0:
                raise ValueError("Must be positive")
            return value

    class FloatSettingMixin(_PositiveValueMixin):
        def _convert(self, value):
            if value:
                return float(value)

    class ByteCountSettingMixin(_PositiveValueMixin):

        _MULTIPLES = {
            # All keys must be the same size.
            'kb': 1024,
            'mb': 1024 * 1024,
            'gb': 1024 * 1024 * 1024,
        }

        _SUFFIX_SIZE = 2

        def _convert(self, value):
            if not value or not isinstance(value, str):
                return value
            value = value.lower()
            for s, m in self._MULTIPLES.items():
                if value[-self._SUFFIX_SIZE:] == s:
                    return int(value[:-self._SUFFIX_SIZE]) * m
            return int(value)

    class Resolver(ImportableSetting, Setting):

        desc = """\
        The callable that will be used to create
        :attr:`gevent.hub.Hub.resolver`.

        See :doc:`dns` for more information.
        """

        default = [
            'thread',
            'dnspython',
            'ares',
            'block',
        ]

        shortname_map = {
            'ares': 'gevent.resolver.ares.Resolver',
            'thread': 'gevent.resolver.thread.Resolver',
            'block': 'gevent.resolver.blocking.Resolver',
            'dnspython': 'gevent.resolver.dnspython.Resolver',
        }

    class Threadpool(ImportableSetting, Setting):

        desc = """\
        The kind of threadpool we use.
        """

        default = 'gevent.threadpool.ThreadPool'

    class ThreadpoolIdleTaskTimeout(FloatSettingMixin, Setting):
        document = True
        name = 'threadpool_idle_task_timeout'
        environment_key = 'GEVENT_THREADPOOL_IDLE_TASK_TIMEOUT'

        desc = """\
        How long threads in the default threadpool (used for
        DNS by default) are allowed to be idle before exiting.

        Use -1 for no timeout.

        .. versionadded:: 22.08.0
        """

        # This value is picked pretty much arbitrarily.
        # We want to balance performance (keeping threads around)
        # with memory/cpu usage (letting threads go).
        default = 5.0

    class Loop(ImportableSetting, Setting):

        desc = """\
        The kind of the loop we use.

        On Windows, this defaults to libuv, while on
        other platforms it defaults to libev.

        """

        default = [
            'libev-cext',
            'libev-cffi',
            'libuv-cffi',
        ] if not WIN else [
            'libuv-cffi',
            'libev-cext',
            'libev-cffi',
        ]

        shortname_map = { # pylint:disable=dict-init-mutate
            'libev-cext': 'gevent.libev.corecext.loop',
            'libev-cffi': 'gevent.libev.corecffi.loop',
            'libuv-cffi': 'gevent.libuv.loop.loop',
        }

        shortname_map['libuv'] = shortname_map['libuv-cffi']

    class FormatContext(ImportableSetting, Setting):
        name = 'format_context'

        # using pprint.pformat can override custom __repr__ methods on dict/list
        # subclasses, which can be a security concern
        default = 'pprint.saferepr'

    class LibevBackend(Setting):
        name = 'libev_backend'
        environment_key = 'GEVENT_BACKEND'

        desc = """\
        The backend for libev, such as 'select'
        """

        default = None

        validate = staticmethod(validate_anything)

    class FileObject(ImportableSetting, Setting):
        desc = """\
        The kind of ``FileObject`` we will use.

        See :mod:`gevent.fileobject` for a detailed description.

        """
        environment_key = 'GEVENT_FILE'

        default = [
            'posix',
            'thread',
        ]

        shortname_map = {
            'thread': 'gevent._fileobjectcommon.FileObjectThread',
            'posix': 'gevent._fileobjectposix.FileObjectPosix',
            'block': 'gevent._fileobjectcommon.FileObjectBlock'
        }

    class WatchChildren(BoolSettingMixin, Setting):
        desc = """\
        Should we *not* watch children with the event loop watchers?

        This is an advanced setting.

        See :mod:`gevent.os` for a detailed description.
        """
        name = 'disable_watch_children'
        environment_key = 'GEVENT_NOWAITPID'
        default = False

    class TraceMalloc(IntSettingMixin, Setting):
        name = 'trace_malloc'
        environment_key = 'PYTHONTRACEMALLOC'
        default = False

        desc = """\
        Should FFI objects track their allocation?

        This is only useful for low-level debugging.

        On Python 3, this environment variable is built in to the
        interpreter, and it may also be set with the ``-X
        tracemalloc`` command line argument.

        On Python 2, gevent interprets this argument and adds extra
        tracking information for FFI objects.
        """

    class TrackGreenletTree(BoolSettingMixin, Setting):
        name = 'track_greenlet_tree'
        environment_key = 'GEVENT_TRACK_GREENLET_TREE'
        default = True

        desc = """\
        Should `Greenlet` objects track their spawning tree?

        Setting this to a false value will make spawning `Greenlet`
        objects and using `spawn_raw` faster, but the
        ``spawning_greenlet``, ``spawn_tree_locals`` and ``spawning_stack``
        will not be captured. Setting this to a false value can also
        reduce memory usage because capturing the stack captures
        some information about Python frames.

        .. versionadded:: 1.3b1
        """

    ## Monitoring settings
    # All env keys should begin with GEVENT_MONITOR

    class MonitorThread(BoolSettingMixin, Setting):
        name = 'monitor_thread'
        environment_key = 'GEVENT_MONITOR_THREAD_ENABLE'
        default = False

        desc = """\
        Should each hub start a native OS thread to monitor
        for problems?

        Such a thread will periodically check to see if the event loop
        is blocked for longer than `max_blocking_time`, producing output on
        the hub's exception stream (stderr by default) if it detects this condition.

        If this setting is true, then this thread will be created
        the first time the hub is switched to,
        or you can call :meth:`gevent.hub.Hub.start_periodic_monitoring_thread` at any
        time to create it (from the same thread that will run the hub). That function
        will return an instance of :class:`gevent.events.IPeriodicMonitorThread`
        to which you can add your own monitoring functions. That function
        also emits an event of :class:`gevent.events.PeriodicMonitorThreadStartedEvent`.

        .. seealso:: `max_blocking_time`

        .. versionadded:: 1.3b1
        """

    class MaxBlockingTime(FloatSettingMixin, Setting):
        name = 'max_blocking_time'
        # This environment key doesn't follow the convention because it's
        # meant to match a key used by existing projects
        environment_key = 'GEVENT_MAX_BLOCKING_TIME'
        default = 0.1

        desc = """\
        If the `monitor_thread` is enabled, this is
        approximately how long (in seconds)
        the event loop will be allowed to block before a warning is issued.

        This function depends on using `greenlet.settrace`, so installing
        your own trace function after starting the monitoring thread will
        cause this feature to misbehave unless you call the function
        returned by `greenlet.settrace`. If you install a tracing function *before*
        the monitoring thread is started, it will still be called.

        .. note:: In the unlikely event of creating and using multiple different
            gevent hubs in the same native thread in a short period of time,
            especially without destroying the hubs, false positives may be reported.

        .. versionadded:: 1.3b1
        """

    class MonitorMemoryPeriod(FloatSettingMixin, Setting):
        name = 'memory_monitor_period'

        environment_key = 'GEVENT_MONITOR_MEMORY_PERIOD'
        default = 5

        desc = """\
        If `monitor_thread` is enabled, this is approximately how long
        (in seconds) we will go between checking the processes memory usage.

        Checking the memory usage is relatively expensive on some operating
        systems, so this should not be too low. gevent will place a floor
        value on it.
        """

    class MonitorMemoryMaxUsage(ByteCountSettingMixin, Setting):
        name = 'max_memory_usage'

        environment_key = 'GEVENT_MONITOR_MEMORY_MAX'
        default = None

        desc = """\
        If `monitor_thread` is enabled,
        then if memory usage exceeds this amount (in bytes), events will
        be emitted. See `gevent.events`. In the environment variable, you can use
        a suffix of 'kb', 'mb' or 'gb' to specify the value in kilobytes, megabytes
        or gigibytes.

        There is no default value for this setting. If you wish to
        cap memory usage, you must choose a value.
        """

    # The ares settings are all interpreted by
    # gevent/resolver/ares.pyx, so we don't do
    # any validation here.

    class AresSettingMixin(object):

        document = False

        @property
        def kwarg_name(self):
            return self.name[5:]

        validate = staticmethod(validate_anything)

        _convert = staticmethod(convert_str_value_as_is)

    class AresFlags(AresSettingMixin, Setting):
        name = 'ares_flags'
        default = None
        environment_key = 'GEVENTARES_FLAGS'

    class AresTimeout(AresSettingMixin, Setting):
        document = True
        name = 'ares_timeout'
        default = None
        environment_key = 'GEVENTARES_TIMEOUT'
        desc = """\

        .. deprecated:: 1.3a2
           Prefer the :attr:`resolver_timeout` setting. If both are set,
           the results are not defined.
        """

    class AresTries(AresSettingMixin, Setting):
        name = 'ares_tries'
        default = None
        environment_key = 'GEVENTARES_TRIES'

    class AresNdots(AresSettingMixin, Setting):
        name = 'ares_ndots'
        default = None
        environment_key = 'GEVENTARES_NDOTS'

    class AresUDPPort(AresSettingMixin, Setting):
        name = 'ares_udp_port'
        default = None
        environment_key = 'GEVENTARES_UDP_PORT'

    class AresTCPPort(AresSettingMixin, Setting):
        name = 'ares_tcp_port'
        default = None
        environment_key = 'GEVENTARES_TCP_PORT'

    class AresServers(AresSettingMixin, Setting):
        document = True
        name = 'ares_servers'
        default = None
        environment_key = 'GEVENTARES_SERVERS'
        desc = """\
        A list of strings giving the IP addresses of nameservers for the ares resolver.

        In the environment variable, these strings are separated by commas.

        .. deprecated:: 1.3a2
           Prefer the :attr:`resolver_nameservers` setting. If both are set,
           the results are not defined.
        """

    # Generic nameservers, works for dnspython and ares.
    class ResolverNameservers(AresSettingMixin, Setting):
        document = True
        name = 'resolver_nameservers'
        default = None
        environment_key = 'GEVENT_RESOLVER_NAMESERVERS'
        desc = """\
        A list of strings giving the IP addresses of nameservers for the (non-system) resolver.

        In the environment variable, these strings are separated by commas.

        .. rubric:: Resolver Behaviour

        * blocking

          Ignored

        * Threaded

          Ignored

        * dnspython

          If this setting is not given, the dnspython resolver will
          load nameservers to use from ``/etc/resolv.conf``
          or the Windows registry. This setting replaces any nameservers read
          from those means. Note that the file and registry are still read
          for other settings.

          .. caution:: dnspython does not validate the members of the list.
             An improper address (such as a hostname instead of IP) has
             undefined results, including hanging the process.

        * ares

          Similar to dnspython, but with more platform and compile-time
          options. ares validates that the members of the list are valid
          addresses.
        """

        # Normal string-to-list rules. But still validate_anything.
        _convert = Setting._convert

        # TODO: In the future, support reading a resolv.conf file
        # *other* than /etc/resolv.conf, and do that both on Windows
        # and other platforms. Also offer the option to disable the system
        # configuration entirely.

        @property
        def kwarg_name(self):
            return 'servers'

    # Generic timeout, works for dnspython and ares
    class ResolverTimeout(FloatSettingMixin, AresSettingMixin, Setting):
        document = True
        name = 'resolver_timeout'
        environment_key = 'GEVENT_RESOLVER_TIMEOUT'
        desc = """\
        The total amount of time that the DNS resolver will spend making queries.

        Only the ares and dnspython resolvers support this.

        .. versionadded:: 1.3a2
        """

        @property
        def kwarg_name(self):
            return 'timeout'

    config = Config()

    # Go ahead and attempt to import the loop when this class is
    # instantiated. The hub won't work if the loop can't be found. This
    # can solve problems with the class being imported from multiple
    # threads at once, leading to one of the imports failing.
    # factories are themselves handled lazily. See #687.

    # Don't cache it though, in case the user re-configures through the
    # API.

    try:
        Loop().get()
    except ImportError: # pragma: no cover
        pass

    if __name__ == '__main__':
        config.print_help()