# Source code for scrapy.utils.python

    """
    This module contains essential stuff that should've come with Python itself ;)
    """

    from __future__ import annotations

    import gc
    import inspect
    import re
    import sys
    import warnings
    import weakref
    from collections.abc import AsyncIterator, Iterable, Mapping
    from functools import partial, wraps
    from itertools import chain
    from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, overload

    from scrapy.exceptions import ScrapyDeprecationWarning
    from scrapy.utils.asyncgen import as_async_generator

    if TYPE_CHECKING:
        from collections.abc import Callable, Iterator
        from re import Pattern

        # typing.Self requires Python 3.11
        from typing_extensions import Self

    _T = TypeVar("_T")
    _KT = TypeVar("_KT")
    _VT = TypeVar("_VT")
    _P = ParamSpec("_P")

    def is_listlike(x: Any) -> bool:
        """
        >>> is_listlike("foo")
        False
        >>> is_listlike(5)
        False
        >>> is_listlike(b"foo")
        False
        >>> is_listlike([b"foo"])
        True
        >>> is_listlike((b"foo",))
        True
        >>> is_listlike({})
        True
        >>> is_listlike(set())
        True
        >>> is_listlike((x for x in range(3)))
        True
        >>> is_listlike(range(5))
        True
        """
        return hasattr(x, "__iter__") and not isinstance(x, (str, bytes))

    def unique(list_: Iterable[_T], key: Callable[[_T], Any] = lambda x: x) -> list[_T]:
        """efficient function to uniquify a list preserving item order"""
        seen = set()
        result: list[_T] = []
        for item in list_:
            seenkey = key(item)
            if seenkey in seen:
                continue
            seen.add(seenkey)
            result.append(item)
        return result

    def to_unicode(
        text: str | bytes, encoding: str | None = None, errors: str = "strict"
    ) -> str:
        """Return the unicode representation of a bytes object ``text``. If
        ``text`` is already an unicode object, return it as-is."""
        if isinstance(text, str):
            return text
        if not isinstance(text, (bytes, str)):
            raise TypeError(
                f"to_unicode must receive a bytes or str object, got {type(text).__name__}"
            )
        if encoding is None:
            encoding = "utf-8"
        return text.decode(encoding, errors)

    def to_bytes(
        text: str | bytes, encoding: str | None = None, errors: str = "strict"
    ) -> bytes:
        """Return the binary representation of ``text``. If ``text``
        is already a bytes object, return it as-is."""
        if isinstance(text, bytes):
            return text
        if not isinstance(text, str):
            raise TypeError(
                f"to_bytes must receive a str or bytes object, got {type(text).__name__}"
            )
        if encoding is None:
            encoding = "utf-8"
        return text.encode(encoding, errors)

    def _chunk_iter(text: str, chunk_size: int) -> Iterable[tuple[str, int]]:
        offset = len(text)
        while True:
            offset -= chunk_size * 1024
            if offset <= 0:
                break
            yield (text[offset:], offset)
        yield (text, 0)

    def re_rsearch(
        pattern: str | Pattern[str], text: str, chunk_size: int = 1024
    ) -> tuple[int, int] | None:
        """
        This function does a reverse search in a text using a regular expression
        given in the attribute 'pattern'.
        Since the re module does not provide this functionality, we have to find for
        the expression into chunks of text extracted from the end (for the sake of efficiency).
        At first, a chunk of 'chunk_size' kilobytes is extracted from the end, and searched for
        the pattern. If the pattern is not found, another chunk is extracted, and another
        search is performed.
        This process continues until a match is found, or until the whole file is read.
        In case the pattern wasn't found, None is returned, otherwise it returns a tuple containing
        the start position of the match, and the ending (regarding the entire text).
        """

        if isinstance(pattern, str):
            pattern = re.compile(pattern)

        for chunk, offset in _chunk_iter(text, chunk_size):
            matches = list(pattern.finditer(chunk))
            if matches:
                start, end = matches[-1].span()
                return offset + start, offset + end
        return None

    _SelfT = TypeVar("_SelfT")

    def memoizemethod_noargs(
        method: Callable[Concatenate[_SelfT, _P], _T],
    ) -> Callable[Concatenate[_SelfT, _P], _T]:
        """Decorator to cache the result of a method (without arguments) using a
        weak reference to its object
        """
        cache: weakref.WeakKeyDictionary[_SelfT, _T] = weakref.WeakKeyDictionary()

        @wraps(method)
        def new_method(self: _SelfT, *args: _P.args, **kwargs: _P.kwargs) -> _T:
            if self not in cache:
                cache[self] = method(self, *args, **kwargs)
            return cache[self]

        return new_method

    _BINARYCHARS = {
        i for i in range(32) if to_bytes(chr(i)) not in {b"\0", b"\t", b"\n", b"\r"}
    }

    def binary_is_text(data: bytes) -> bool:
        """Returns ``True`` if the given ``data`` argument (a ``bytes`` object)
        does not contain unprintable control characters.
        """
        if not isinstance(data, bytes):
            raise TypeError(f"data must be bytes, got '{type(data).__name__}'")
        return all(c not in _BINARYCHARS for c in data)

    def get_func_args_dict(
        func: Callable[..., Any], stripself: bool = False
    ) -> Mapping[str, inspect.Parameter]:
        """Return the argument dict of a callable object.

        .. versionadded:: 2.14
        """
        if not callable(func):
            raise TypeError(f"func must be callable, got '{type(func).__name__}'")

        args: Mapping[str, inspect.Parameter]
        try:
            sig = inspect.signature(func)
        except ValueError:
            return {}

        if isinstance(func, partial):
            partial_args = func.args
            partial_kw = func.keywords

            args = {}
            for name, param in sig.parameters.items():
                if name in partial_args:
                    continue
                if partial_kw and name in partial_kw:
                    continue
                args[name] = param
        else:
            args = sig.parameters

        if stripself and args and "self" in args:
            args = {k: v for k, v in args.items() if k != "self"}
        return args

    def get_func_args(func: Callable[..., Any], stripself: bool = False) -> list[str]:
        """Return the argument name list of a callable object"""
        return list(get_func_args_dict(func, stripself=stripself))

    def get_spec(func: Callable[..., Any]) -> tuple[list[str], dict[str, Any]]:
        """Returns (args, kwargs) tuple for a function
        >>> import re
        >>> get_spec(re.match)
        (['pattern', 'string'], {'flags': 0})

        >>> class Test:
        ...     def __call__(self, val):
        ...         pass
        ...     def method(self, val, flags=0):
        ...         pass

        >>> get_spec(Test)
        (['self', 'val'], {})

        >>> get_spec(Test.method)
        (['self', 'val'], {'flags': 0})

        >>> get_spec(Test().method)
        (['self', 'val'], {'flags': 0})
        """

        if inspect.isfunction(func) or inspect.ismethod(func):
            spec = inspect.getfullargspec(func)
        elif hasattr(func, "__call__"):  # noqa: B004
            spec = inspect.getfullargspec(func.__call__)
        else:
            raise TypeError(f"{type(func)} is not callable")

        defaults: tuple[Any, ...] = spec.defaults or ()

        firstdefault = len(spec.args) - len(defaults)
        args = spec.args[:firstdefault]
        kwargs = dict(zip(spec.args[firstdefault:], defaults, strict=False))
        return args, kwargs

    @overload
    def without_none_values(iterable: Mapping[_KT, _VT]) -> dict[_KT, _VT]: ...

    @overload
    def without_none_values(iterable: Iterable[_KT]) -> Iterable[_KT]: ...

    def without_none_values(
        iterable: Mapping[_KT, _VT] | Iterable[_KT],
    ) -> dict[_KT, _VT] | Iterable[_KT]:
        """Return a copy of ``iterable`` with all ``None`` entries removed.

        If ``iterable`` is a mapping, return a dictionary where all pairs that have
        value ``None`` have been removed.
        """
        if isinstance(iterable, Mapping):
            return {k: v for k, v in iterable.items() if v is not None}
        # the iterable __init__ must take another iterable
        return type(iterable)(v for v in iterable if v is not None)  # type: ignore[call-arg]

    def global_object_name(obj: Any) -> str:
        """Return the full import path of the given object.

        >>> from scrapy import Request
        >>> global_object_name(Request)
        'scrapy.http.request.Request'
        >>> global_object_name(Request.replace)
        'scrapy.http.request.Request.replace'
        """
        return f"{obj.__module__}.{obj.__qualname__}"

    if hasattr(sys, "pypy_version_info"):

        def garbage_collect() -> None:
            # Collecting weakreferences can take two collections on PyPy.
            gc.collect()
            gc.collect()

    else:

        def garbage_collect() -> None:
            gc.collect()

    class MutableChain(Iterable[_T]):  # pragma: no cover
        def __init__(self, *args: Iterable[_T]):
            warnings.warn(
                "MutableChain is deprecated and will be removed in a future Scrapy version.",
                category=ScrapyDeprecationWarning,
                stacklevel=2,
            )
            self.data: Iterator[_T] = chain.from_iterable(args)

        def extend(self, *iterables: Iterable[_T]) -> None:
            self.data = chain(self.data, chain.from_iterable(iterables))

        def __iter__(self) -> Iterator[_T]:
            return self

        def __next__(self) -> _T:
            return next(self.data)

    async def _async_chain(
        *iterables: Iterable[_T] | AsyncIterator[_T],
    ) -> AsyncIterator[_T]:
        for it in iterables:
            async for o in as_async_generator(it):
                yield o

    class MutableAsyncChain(AsyncIterator[_T]):
        """
        Similar to MutableChain but for async iterables
        """

        def __init__(self, *args: Iterable[_T] | AsyncIterator[_T]):
            self.data: AsyncIterator[_T] = _async_chain(*args)

        def extend(self, *iterables: Iterable[_T] | AsyncIterator[_T]) -> None:
            self.data = _async_chain(self.data, _async_chain(*iterables))

        def __aiter__(self) -> Self:
            return self

        async def __anext__(self) -> _T:
            return await self.data.__anext__()

    def _looks_like_import_path(value: str) -> bool:
        """Return True if **value** looks like a valid Python import path or False
        otherwise."""
        if not value:
            return False
        if any(c.isspace() for c in value):
            return False
        allowed_chars = set(
            "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_."
        )
        if any(c not in allowed_chars for c in value):
            return False
        if value[0] == "." or value[-1] == ".":
            return False
        parts = value.split(".")
        if any(part == "" for part in parts):
            return False
        return all(part.isidentifier() for part in parts)

    def _iter_exc_causes(exc: BaseException) -> Iterable[BaseException]:
        """Iterate over the exception causes/contexts."""
        seen: set[int] = set()
        cur: BaseException | None = exc
        while cur is not None and id(cur) not in seen:
            seen.add(id(cur))
            yield cur
            cur = cur.__cause__ or cur.__context__