Hide navigation sidebar
Hide table of contents sidebar
Skip to content
Back to top
Toggle table of contents sidebar

# Source code for gevent.threading

    """
    Implementation of the standard :mod:`threading` using greenlets.

    .. note::

        This module is a helper for :mod:`gevent.monkey` and is not
        intended to be used directly. For spawning greenlets in your
        applications, prefer higher level constructs like
        :class:`gevent.Greenlet` class or :func:`gevent.spawn`. Attributes
        in this module like ``__threading__`` are implementation artifacts subject
        to change at any time.

    .. versionchanged:: 1.2.3

       Defer adjusting the stdlib's list of active threads until we are
       monkey patched. Previously this was done at import time. We are
       documented to only be used as a helper for monkey patching, so this should
       functionally be the same, but some applications ignore the documentation and
       directly import this module anyway.

       A positive consequence is that ``import gevent.threading,
       threading; threading.current_thread()`` will no longer return a DummyThread
       before monkey-patching.
    """

    import os
    import sys

    __implements__ = [
        'local',
        '_allocate_lock',
        'Lock',
        '_get_ident',
        '_sleep',
        '_DummyThread',
        # RLock cannot go here, even though we need to import it.
        # If it goes here, it replaces the RLock from the native
        # threading module, but we really just need it here when some
        # things import this module.
        #'RLock',
    ] + ([
        '_start_new_thread',
    ] if sys.version_info[:2] < (3, 13) else [
        '_start_joinable_thread',
        '_ThreadHandle',
        '_make_thread_handle',
    ])

    __extensions__ = [
    ]

    import threading as __threading__ # imports os, sys, _thread, functools, time, itertools
    _DummyThread_ = __threading__._DummyThread
    _MainThread_ = __threading__._MainThread

    from gevent.local import local
    from gevent.thread import start_new_thread as _start_new_thread
    from gevent.thread import start_joinable_thread
    from gevent.thread import _ThreadHandle
    from gevent.thread import _make_thread_handle
    from gevent.thread import allocate_lock as _allocate_lock
    from gevent.thread import get_ident as _get_ident
    from gevent.hub import sleep as _sleep, getcurrent
    from gevent.lock import RLock

    from gevent._util import LazyOnClass

    # Exports, prevent unused import warnings.
    # XXX: Why don't we use __all__?
    local = local
    start_new_thread = _start_new_thread
    _start_joinable_thread = start_joinable_thread
    _make_thread_handle = _make_thread_handle
    _ThreadHandle = _ThreadHandle
    allocate_lock = _allocate_lock
    _get_ident = _get_ident
    _sleep = _sleep
    getcurrent = getcurrent

    Lock = _allocate_lock
    RLock = RLock

    def _cleanup(g):
        __threading__._active.pop(_get_ident(g), None)

    def _make_cleanup_id(gid):
        def _(_r):
            __threading__._active.pop(gid, None)
        return _

    _weakref = None

    class _DummyThread(_DummyThread_):
        # We avoid calling the superclass constructor. This makes us about
        # twice as fast:
        #
        # - 1.16 vs 0.68usec on PyPy (unknown version, older Intel mac)
        # - 29.3 vs 17.7usec on CPython 2.7 (older intel Mac)
        # - 0.98 vs 2.95usec on CPython 3.12.2 (newer M2 mac)
        #
        # It als has the important effect of avoiding allocation and then
        # immediate deletion of _Thread__block, a lock. This is especially
        # important on PyPy where locks go through the cpyext API and
        # Cython, which is known to be slow and potentially buggy (e.g.,
        # https://bitbucket.org/pypy/pypy/issues/2149/memory-leak-for-python-subclass-of-cpyext#comment-22347393)

        # These objects are constructed quite frequently in some cases, so
        # the optimization matters: for example, in gunicorn, which uses
        # pywsgi.WSGIServer, most every request is handled in a new greenlet,
        # and every request uses a logging.Logger to write the access log,
        # and every call to a log method captures the current thread (by
        # default).
        #
        # (Obviously we have to duplicate the effects of the constructor,
        # at least for external state purposes, which is potentially
        # slightly fragile.)

        # For the same reason, instances of this class will cleanup their own entry
        # in ``threading._active``

        # This class also solves a problem forking process with subprocess: after forking,
        # Thread.__stop is called, which throws an exception when __block doesn't
        # exist.

        # Capture the static things as class vars to save on memory/
        # construction time.
        # In Py2, they're all private; in Py3, they become protected
        _Thread__stopped = _is_stopped = _stopped = False
        _Thread__initialized = _initialized = True
        _Thread__daemonic = _daemonic = True
        _Thread__args = _args = ()
        _Thread__kwargs = _kwargs = None
        _Thread__target = _target = None
        _Thread_ident = _ident = None
        _Thread__started = _started = __threading__.Event()
        _Thread__started.set()
        _tstate_lock = None
        _handle = None # 3.13

        def __init__(self): # pylint:disable=super-init-not-called
            #_DummyThread_.__init__(self)

            # It'd be nice to use a pattern like "greenlet-%d", but there are definitely
            # third-party libraries checking thread names to detect DummyThread objects.
            self._name = self._Thread__name = __threading__._newname("Dummy-%d")
            # All dummy threads in the same native thread share the same ident
            # (that of the native thread), unless we're monkey-patched.
            self._set_ident()
            # _handle is only needed for 3.13; keeps a weak reference
            # to the greenlet.
            self._handle = _make_thread_handle(self._ident)
            # ``_native_id`` backs the ``native_id`` property.
            self._native_id = __threading__.get_native_id()

            g = getcurrent()
            gid = _get_ident(g)
            __threading__._active[gid] = self
            rawlink = getattr(g, 'rawlink', None)
            if rawlink is not None:
                # raw greenlet.greenlet greenlets don't
                # have rawlink...
                rawlink(_cleanup)
            else:
                # ... so for them we use weakrefs.
                # See https://github.com/gevent/gevent/issues/918
                ref = self.__weakref_ref
                ref = ref(g, _make_cleanup_id(gid)) # pylint:disable=too-many-function-args
                self.__raw_ref = ref
                assert self.__raw_ref is ref # prevent pylint thinking its unused

        def _Thread__stop(self):
            pass

        _stop = _Thread__stop # py3

        def _wait_for_tstate_lock(self, *args, **kwargs): # pylint:disable=signature-differs
            pass

        @LazyOnClass
        def __weakref_ref(self):
            return __import__('weakref').ref

        # In Python 3.11.8+ and 3.12.2+ (yes, minor patch releases),
        # CPython's ``threading._after_fork`` hook began swizzling the
        # type of the _DummyThread into _MainThread if such a dummy thread
        # was the current thread when ``os.fork()`` gets called.
        # From CPython's perspective, that's a more-or-less fine thing to do.
        # While _DummyThread isn't a subclass of _MainThread, they are both
        # subclasses of Thread, and _MainThread doesn't add any new instance
        # variables.
        #
        # From gevent's perspective, that's NOT good. Our _DummyThread
        # doesn't have all the instance variables that Thread does, and so
        # attempting to do anything with this now-fake _MainThread doesn't work.
        # You in fact immediately get assertion errors from inside ``_after_fork``.
        # Now, these are basically harmless ---  they're printed, and they prevent the cleanup
        # of some globals in _threading, but that probably doesn't matter --- but
        # people complained, and it could break some test scenarios (due to unexpected
        # output on stderr, for example)
        #
        # We thought of a few options to patch around this:
        #
        # - Live with the performance penalty. Newer CPythons are making it
        #   harder and harder to perform well, so if we can possibly avoid
        #   adding our own performance regressions, that would be good.
        #
        # - ``after_fork`` uses ``isinstance(current, _DummyThread)``
        #   before swizzling, so we could use a metaclass to make that
        #   check return false. That's a fairly large compatibility risk,
        #   both because of the use of a metaclass (what if some other
        #   subclass of _DummyTHread is using an incompatible metaclass?)
        #   and the change in ``isinstance`` behaviour. We could limit the latter
        #   to a window around the fork, using ``os.register_at_fork(before, after_in_parent=)``,
        #   but that's a lot of moving pieces requiring the use of a global or class
        #   variable to track state.
        #
        # - We could copy the ivars of the current main thread into the
        #   _DummyThread in ``register_at_fork(before=)``. That appears to
        #   work, but also requires the use of
        #   ``register_at_fork(after_in_parent=)`` to reverse it.
        #
        # - We could simply prevent swizzling the class in the first
        #   place. In combination with
        #   ``register_at_fork(after_in_child=)`` to establish a *real*
        #   new _MainThread, that's a clean solution. Establishing a real
        #   new _MainThread is something that CPython itself is prepared
        #   to do if it can't figure out what the current thread is. The
        #   compatibility risk of this is relatively low: swizzling
        #   classes is frowned upon and uncommon, and we can limit it to
        #   just preventing this specific case. And if somebody was
        #   attempting this already with some other thread subclass, it
        #   would (probably?) have the exact same issues, so we can be pretty
        #   sure nobody is doing that.
        #
        # We're initially going with the last fix; the __class__ part is here,
        # the ``after_in_child`` fixup we only apply if we're monkey-patching.
        #
        # Now, all of this is moot in 3.13, which takes a very different
        # approach to handling this, and also changes some names. See
        # https://github.com/python/cpython/commit/0e9c364f4ac18a2237bdbac702b96bcf8ef9cb09

        # Tests pass just fine in 3.8 (and presumably 3.9 and 3.10) with these fixes
        # applied, but just in case, we only do it where we know it's necessary.
        _NEEDS_CLASS_FORK_FIXUP = (
            (sys.version_info[:2] == (3, 11) and sys.version_info[:3] >= (3, 11, 8))
            or sys.version_info[:3] >= (3, 12, 2)
        )

        if _NEEDS_CLASS_FORK_FIXUP:
            # Override with a property, as opposed to using __setattr__,
            # to avoid adding overhead on any other attribute setting.
            @property
            def __class__(self):
                return type(self)

            @__class__.setter
            def __class__(self, new_class):
                # Even if we wanted to allow setting this, I'm not sure
                # exactly how to do so when we have a property object handling it.
                # Getting the descriptor from ``object.__dict__['__class__']``
                # and using its ``__set__`` method raises a TypeError (as does
                # the simpler ``super().__class__``).
                #
                # Better allow the TypeError for now as opposed to silently ignoring
                # the assignment.
                if new_class is not _MainThread_:
                    object.__dict__['__class__'].__set__(self, new_class)

    def main_native_thread():
        return __threading__.main_thread() # pylint:disable=no-member

    # XXX: Issue 18808 breaks us on Python 3.4+.
    # Thread objects now expect a callback from the interpreter itself
    # (threadmodule.c:release_sentinel) when the C-level PyThreadState
    # object is being deallocated. Because this never happens
    # when a greenlet exits, join() and friends will block forever.
    # Fortunately this is easy to fix: just ensure that the allocation of the
    # lock, _set_sentinel, creates a *gevent* lock, and release it when
    # we're done. The main _shutdown code is in Python and deals with
    # this gracefully.

    class Thread(__threading__.Thread):

        # Only happens in < 3.13
        def _set_tstate_lock(self):
            super(Thread, self)._set_tstate_lock()
            greenlet = getcurrent()
            greenlet.rawlink(self.__greenlet_finished)

        def __greenlet_finished(self, _):
            if self._tstate_lock:
                self._tstate_lock.release()
                self._stop()

    __implements__.append('Thread')

    class Timer(Thread, __threading__.Timer): # pylint:disable=abstract-method,inherit-non-class
        pass

    __implements__.append('Timer')

    _set_sentinel = allocate_lock
    if sys.version_info[:2] < (3, 13):
        __implements__.append('_set_sentinel')
    else:
        __extensions__.append('_set_sentinel')
    # The main thread is patched up with more care
    # in _gevent_will_monkey_patch

    __implements__.remove('_get_ident')
    __implements__.append('get_ident')
    get_ident = _get_ident
    __implements__.remove('_sleep')

    if hasattr(__threading__, '_CRLock'):
        # Python 3 changed the implementation of threading.RLock
        # Previously it was a factory function around threading._RLock
        # which in turn used _allocate_lock. Now, it wants to use
        # threading._CRLock, which is imported from _thread.RLock and as such
        # is implemented in C. So it bypasses our _allocate_lock function.
        # Fortunately they left the Python fallback in place and use it
        # if the imported _CRLock is None; this arranges for that to be the case.

        # This was also backported to PyPy 2.7-7.0
        _CRLock = None
        __implements__.append('_CRLock')

    class _ForkHooks:

        _before_fork_current_thread = None
        _before_fork_active = None

        def before_fork_in_parent(self):
            self._before_fork_active = dict(__threading__._active)
            self._before_fork_current_thread = __threading__.current_thread()

        def after_fork_in_child(self):
            # We've already imported threading, which installed its "after" hook,
            # so we're going to be called after that hook.
            # Note that this is only installed when monkey-patching.
            # TODO: Is there any point to checking to see if the current thread is
            # our dummy thread before doing this?
            active = __threading__._active
            assert len(active) == 1

            # We cannot actually kill the greenlets via throw():
            # - If it was the main greenlet, the process will exit
            # - In any case, that would unwind the stack and execute
            #   code that, before 2024-10-03, would never have executed.
            #
            # But we can take them out of the map and make them appear
            # stopped; if they truly are no longer referenced, GC will
            # kick in a nd delete the greenlet. If gevent is still waiting
            # to switch to them, that will still happen...
            #
            # This happens automatically on <= 3.12 which hardcodes the call to
            # ``Thread._stop``; for 3.13, we need to go through the handle.
            current_ident = get_ident()
            for green_ident, thread in self._before_fork_active.items():
                if green_ident != current_ident:
                    try:
                        handle = thread._handle
                    except AttributeError:
                        assert sys.version_info[:2] < (3, 13)
                        assert not thread.is_alive()
                    else:
                        # We DO NOT want to bounce to the hub. We're running
                        # at a very sensitive time and it's best to keep tight control
                        # over what gets to run.
                        handle._set_done(enter_hub=False)

            main = __threading__._MainThread()
            main._ident = get_ident() # 3.13: reset to the greenlet version.
            __threading__._active[__threading__.get_ident()] = main
            __threading__._main_thread = main

            main = __threading__.main_thread()
            # XXX: Not the case. Maybe don't save main.
            assert main.ident == __threading__.get_ident()

    _fork_hooks = _ForkHooks()

    def _gevent_will_monkey_patch(native_module, items, warn): # pylint:disable=unused-argument
        # Make sure the MainThread can be found by our current greenlet ID,
        # otherwise we get a new DummyThread, which cannot be joined.
        # Fixes tests in test_threading_2 under PyPy.
        main_thread = main_native_thread()
        if __threading__.current_thread() != main_thread:
            warn("Monkey-patching outside the main native thread. Some APIs "
                 "will not be available. Expect a KeyError to be printed at shutdown.")
            return

        if _get_ident() not in __threading__._active:
            main_id = main_thread.ident
            del __threading__._active[main_id]
            main_thread._ident = main_thread._Thread__ident = _get_ident()
            __threading__._active[_get_ident()] = main_thread

        register_at_fork = getattr(os, 'register_at_fork', None)
        if register_at_fork:
            #if _DummyThread._NEEDS_CLASS_FORK_FIXUP:
            register_at_fork(
                before=_fork_hooks.before_fork_in_parent,
                after_in_child=_fork_hooks.after_fork_in_child)