# Source code for persistent.persistence

    ##############################################################################
    #
    # Copyright (c) 2011 Zope Foundation and Contributors.
    # All Rights Reserved.
    #
    # This software is subject to the provisions of the Zope Public License,
    # Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
    # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
    # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
    # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
    # FOR A PARTICULAR PURPOSE.
    #
    ##############################################################################
    import struct

    from zope.interface import implementer

    from persistent.interfaces import IPersistent
    from persistent.interfaces import GHOST
    from persistent.interfaces import UPTODATE
    from persistent.interfaces import CHANGED
    from persistent.interfaces import STICKY

    from persistent.interfaces import SERIAL_TYPE
    from persistent.timestamp import TimeStamp
    from persistent.timestamp import _ZERO
    from persistent._compat import copy_reg
    from persistent._compat import intern

    _INITIAL_SERIAL = _ZERO

    # Bitwise flags
    _CHANGED = 0x0001
    _STICKY = 0x0002

    _OGA = object.__getattribute__
    _OSA = object.__setattr__
    _ODA = object.__delattr__

    # These names can be used from a ghost without causing it to be
    # activated. These are standardized with the C implementation
    SPECIAL_NAMES = ('__class__',
                     '__del__',
                     '__dict__',
                     '__of__',
                     '__setstate__',)

    # And this is an implementation detail of this class; it holds
    # the standard names plus the slot names, allowing for just one
    # check in __getattribute__
    _SPECIAL_NAMES = set(SPECIAL_NAMES)

    # Represent 8-byte OIDs as hex integer, just like
    # ZODB does.
    _OID_STRUCT = struct.Struct('>Q')
    _OID_UNPACK = _OID_STRUCT.unpack

    @implementer(IPersistent)
    class Persistent(object):
        """ Pure Python implmentation of Persistent base class
        """
        __slots__ = ('__jar', '__oid', '__serial', '__flags', '__size', '__ring',)

        def __new__(cls, *args, **kw):
            inst = super(Persistent, cls).__new__(cls)
            # We bypass the __setattr__ implementation of this object
            # at __new__ time, just like the C implementation does. This
            # makes us compatible with subclasses that want to access
            # properties like _p_changed in their setattr implementation
            _OSA(inst, '_Persistent__jar', None)
            _OSA(inst, '_Persistent__oid', None)
            _OSA(inst, '_Persistent__serial', None)
            _OSA(inst, '_Persistent__flags', None)
            _OSA(inst, '_Persistent__size', 0)
            _OSA(inst, '_Persistent__ring', None)
            return inst

        # _p_jar:  see IPersistent.
        def _get_jar(self):
            return _OGA(self, '_Persistent__jar')

        def _set_jar(self, value):
            jar = _OGA(self, '_Persistent__jar')
            if self._p_is_in_cache(jar) and value is not None and jar != value:
                # The C implementation only forbids changing the jar
                # if we're already in a cache. Match its error message
                raise ValueError('can not change _p_jar of cached object')

            if _OGA(self, '_Persistent__jar') != value:
                _OSA(self, '_Persistent__jar', value)
                _OSA(self, '_Persistent__flags', 0)

        def _del_jar(self):
            jar = _OGA(self, '_Persistent__jar')
            if jar is not None:
                if self._p_is_in_cache(jar):
                    raise ValueError("can't delete _p_jar of cached object")
                _OSA(self, '_Persistent__jar', None)
                _OSA(self, '_Persistent__flags', None)

        _p_jar = property(_get_jar, _set_jar, _del_jar)

        # _p_oid:  see IPersistent.
        def _get_oid(self):
            return _OGA(self, '_Persistent__oid')

        def _set_oid(self, value):
            if value == _OGA(self, '_Persistent__oid'):
                return
            # The C implementation allows *any* value to be
            # used as the _p_oid.
            #if value is not None:
            #    if not isinstance(value, OID_TYPE):
            #        raise ValueError('Invalid OID type: %s' % value)
            # The C implementation only forbids changing the OID
            # if we're in a cache, regardless of what the current
            # value or jar is
            if self._p_is_in_cache():
                # match the C error message
                raise ValueError('can not change _p_oid of cached object')
            _OSA(self, '_Persistent__oid', value)

        def _del_oid(self):
            jar = _OGA(self, '_Persistent__jar')
            oid = _OGA(self, '_Persistent__oid')
            if jar is not None:
                if oid and jar._cache.get(oid):
                    raise ValueError('Cannot delete _p_oid of cached object')
            _OSA(self, '_Persistent__oid', None)

        _p_oid = property(_get_oid, _set_oid, _del_oid)

        # _p_serial:  see IPersistent.
        def _get_serial(self):
            serial = _OGA(self, '_Persistent__serial')
            if serial is not None:
                return serial
            return _INITIAL_SERIAL

        def _set_serial(self, value):
            if not isinstance(value, SERIAL_TYPE):
                raise ValueError('Invalid SERIAL type: %s' % value)
            if len(value) != 8:
                raise ValueError('SERIAL must be 8 octets')
            _OSA(self, '_Persistent__serial', value)

        def _del_serial(self):
            _OSA(self, '_Persistent__serial', None)

        _p_serial = property(_get_serial, _set_serial, _del_serial)

        # _p_changed:  see IPersistent.
        def _get_changed(self):
            if _OGA(self, '_Persistent__jar') is None:
                return False
            flags = _OGA(self, '_Persistent__flags')
            if flags is None: # ghost
                return None
            return bool(flags & _CHANGED)

        def _set_changed(self, value):
            if _OGA(self, '_Persistent__flags') is None:
                if value:
                    self._p_activate()
                    self._p_set_changed_flag(value)
            else:
                if value is None: # -> ghost
                    self._p_deactivate()
                else:
                    self._p_set_changed_flag(value)

        def _del_changed(self):
            self._p_invalidate()

        _p_changed = property(_get_changed, _set_changed, _del_changed)

        # _p_mtime
        def _get_mtime(self):
            # The C implementation automatically unghostifies the object
            # when _p_mtime is accessed.
            self._p_activate()
            self._p_accessed()
            serial = _OGA(self, '_Persistent__serial')
            if serial is not None:
                ts = TimeStamp(serial)
                return ts.timeTime()

        _p_mtime = property(_get_mtime)

        # _p_state
        def _get_state(self):
            # Note the use of OGA and caching to avoid recursive calls to __getattribute__:
            # __getattribute__ calls _p_accessed calls cache.mru() calls _p_state
            if _OGA(self, '_Persistent__jar') is None:
                return UPTODATE
            flags = _OGA(self, '_Persistent__flags')
            if flags is None:
                return GHOST
            if flags & _CHANGED:
                result = CHANGED
            else:
                result = UPTODATE
            if flags & _STICKY:
                return STICKY
            return result

        _p_state = property(_get_state)

        # _p_estimated_size:  XXX don't want to reserve the space?
        def _get_estimated_size(self):
            return _OGA(self, '_Persistent__size') * 64

        def _set_estimated_size(self, value):
            if isinstance(value, int):
                if value < 0:
                    raise ValueError('_p_estimated_size must not be negative')
                _OSA(self, '_Persistent__size', _estimated_size_in_24_bits(value))
            else:
                raise TypeError("_p_estimated_size must be an integer")

        def _del_estimated_size(self):
            _OSA(self, '_Persistent__size', 0)

        _p_estimated_size = property(
            _get_estimated_size, _set_estimated_size, _del_estimated_size)

        # The '_p_sticky' property is not (yet) part of the API:  for now,
        # it exists to simplify debugging and testing assertions.
        def _get_sticky(self):
            flags = _OGA(self, '_Persistent__flags')
            if flags is None:
                return False
            return bool(flags & _STICKY)
        def _set_sticky(self, value):
            flags = _OGA(self, '_Persistent__flags')
            if flags is None:
                raise ValueError('Ghost')
            if value:
                flags |= _STICKY
            else:
                flags &= ~_STICKY
            _OSA(self, '_Persistent__flags', flags)
        _p_sticky = property(_get_sticky, _set_sticky)

        # The '_p_status' property is not (yet) part of the API:  for now,
        # it exists to simplify debugging and testing assertions.
        def _get_status(self):
            if _OGA(self, '_Persistent__jar') is None:
                return 'unsaved'
            flags = _OGA(self, '_Persistent__flags')
            if flags is None:
                return 'ghost'
            if flags & _STICKY:
                return 'sticky'
            if flags & _CHANGED:
                return 'changed'
            return 'saved'

        _p_status = property(_get_status)

        # Methods from IPersistent.
        def __getattribute__(self, name):
            """ See IPersistent.
            """
            oga = _OGA
            if (not name.startswith('_p_') and
                name not in _SPECIAL_NAMES):
                if oga(self, '_Persistent__flags') is None:
                    oga(self, '_p_activate')()
                oga(self, '_p_accessed')()
            return oga(self, name)

        def __setattr__(self, name, value):
            special_name = (name in _SPECIAL_NAMES or
                            name.startswith('_p_'))
            volatile = name.startswith('_v_')
            if not special_name:
                if _OGA(self, '_Persistent__flags') is None:
                    _OGA(self, '_p_activate')()
                if not volatile:
                    _OGA(self, '_p_accessed')()
            _OSA(self, name, value)
            if (_OGA(self, '_Persistent__jar') is not None and
                _OGA(self, '_Persistent__oid') is not None and
                not special_name and
                not volatile):
                before = _OGA(self, '_Persistent__flags')
                after = before | _CHANGED
                if before != after:
                    _OSA(self, '_Persistent__flags', after)
                    _OGA(self, '_p_register')()

        def __delattr__(self, name):
            special_name = (name in _SPECIAL_NAMES or
                            name.startswith('_p_'))
            if not special_name:
                if _OGA(self, '_Persistent__flags') is None:
                    _OGA(self, '_p_activate')()
                _OGA(self, '_p_accessed')()
                before = _OGA(self, '_Persistent__flags')
                after = before | _CHANGED
                if before != after:
                    _OSA(self, '_Persistent__flags', after)
                    if (_OGA(self, '_Persistent__jar') is not None and
                        _OGA(self, '_Persistent__oid') is not None):
                        _OGA(self, '_p_register')()
            _ODA(self, name)

        def _slotnames(self, _v_exclude=True):
            slotnames = copy_reg._slotnames(type(self))
            return [x for x in slotnames
                       if not x.startswith('_p_') and
                          not (x.startswith('_v_') and _v_exclude) and
                          not x.startswith('_Persistent__') and
                          x not in Persistent.__slots__]

        def __getstate__(self):
            """ See IPersistent.
            """
            idict = getattr(self, '__dict__', None)
            slotnames = self._slotnames()
            if idict is not None:
                d = dict([x for x in idict.items()
                             if not x[0].startswith('_p_') and
                                not x[0].startswith('_v_')])
            else:
                d = None
            if slotnames:
                s = {}
                for slotname in slotnames:
                    value = getattr(self, slotname, self)
                    if value is not self:
                        s[slotname] = value
                return d, s
            return d

        def __setstate__(self, state):
            """ See IPersistent.
            """
            if isinstance(state,tuple):
                inst_dict, slots = state
            else:
                inst_dict, slots = state, ()
            idict = getattr(self, '__dict__', None)
            if inst_dict is not None:
                if idict is None:
                    raise TypeError('No instance dict')
                idict.clear()
                for k, v in inst_dict.items():
                    # Normally the keys for instance attributes are interned.
                    # Do that here, but only if it is possible to do so.
                    idict[intern(k) if type(k) is str else k] = v
            slotnames = self._slotnames()
            if slotnames:
                for k, v in slots.items():
                    setattr(self, k, v)

        def __reduce__(self):
            """ See IPersistent.
            """
            gna = getattr(self, '__getnewargs__', lambda: ())
            return (copy_reg.__newobj__,
                    (type(self),) + gna(), self.__getstate__())

        def _p_activate(self):
            """ See IPersistent.
            """
            oga = _OGA
            before = oga(self, '_Persistent__flags')
            if before is None: # Only do this if we're a ghost
                # Begin by marking up-to-date in case we bail early
                _OSA(self, '_Persistent__flags', 0)
                jar = oga(self, '_Persistent__jar')
                if jar is None:
                    return
                oid = oga(self, '_Persistent__oid')
                if oid is None:
                    return

                # If we're actually going to execute a set-state,
                # mark as changed to prevent any recursive call
                # (actually, our earlier check that we're a ghost should
                # prevent this, but the C implementation sets it to changed
                # while calling jar.setstate, and this is observable to clients).
                # The main point of this is to prevent changes made during
                # setstate from registering the object with the jar.
                _OSA(self, '_Persistent__flags', CHANGED)
                try:
                    jar.setstate(self)
                except:
                    _OSA(self, '_Persistent__flags', before)
                    raise
                else:
                    # If we succeed, no matter what the implementation
                    # of setstate did, mark ourself as up-to-date. The
                    # C implementation unconditionally does this.
                    _OSA(self, '_Persistent__flags', 0) # up-to-date

        # In the C implementation, _p_invalidate winds up calling
        # _p_deactivate. There are ZODB tests that depend on this;
        # it's not documented but there may be code in the wild
        # that does as well

        def _p_deactivate(self):
            """ See IPersistent.
            """
            flags = _OGA(self, '_Persistent__flags')
            if flags is not None and not flags:
                self._p_invalidate_deactivate_helper()

        def _p_invalidate(self):
            """ See IPersistent.
            """
            # If we think we have changes, we must pretend
            # like we don't so that deactivate does its job
            _OSA(self, '_Persistent__flags', 0)
            self._p_deactivate()

        def _p_invalidate_deactivate_helper(self, clear=True):
            jar = _OGA(self, '_Persistent__jar')
            if jar is None:
                return

            if _OGA(self, '_Persistent__flags') is not None:
                _OSA(self, '_Persistent__flags', None)

            if clear:
                try:
                    idict = _OGA(self, '__dict__')
                except AttributeError:
                    pass
                else:
                    idict.clear()
                type_ = type(self)
                # for backward-compatibility reason we release __slots__ only if
                # class does not override __new__
                if type_.__new__ is Persistent.__new__:
                    for slotname in Persistent._slotnames(self, _v_exclude=False):
                        try:
                            getattr(type_, slotname).__delete__(self)
                        except AttributeError:
                            # AttributeError means slot variable was not initialized at all -
                            # - we can simply skip its deletion.
                            pass

            # Implementation detail: deactivating/invalidating
            # updates the size of the cache (if we have one)
            # by telling it this object no longer takes any bytes
            # (-1 is a magic number to compensate for the implementation,
            # which always adds one to the size given)
            try:
                cache = jar._cache
            except AttributeError:
                pass
            else:
                cache.update_object_size_estimation(_OGA(self, '_Persistent__oid'), -1)
                # See notes in PickleCache.sweep for why we have to do this
                cache._persistent_deactivate_ran = True

        def _p_getattr(self, name):
            """ See IPersistent.
            """
            if name.startswith('_p_') or name in _SPECIAL_NAMES:
                return True
            self._p_activate()
            self._p_accessed()
            return False

        def _p_setattr(self, name, value):
            """ See IPersistent.
            """
            if name.startswith('_p_'):
                _OSA(self, name, value)
                return True
            self._p_activate()
            self._p_accessed()
            return False

        def _p_delattr(self, name):
            """ See IPersistent.
            """
            if name.startswith('_p_'):
                if name == '_p_oid' and self._p_is_in_cache(_OGA(self, '_Persistent__jar')):
                    # The C implementation forbids deleting the oid
                    # if we're already in a cache. Match its error message
                    raise ValueError('can not change _p_jar of cached object')

                _ODA(self, name)
                return True
            self._p_activate()
            self._p_accessed()
            return False

        # Helper methods:  not APIs:  we name them with '_p_' to bypass
        # the __getattribute__ bit which bumps the cache.
        def _p_register(self):
            jar = _OGA(self, '_Persistent__jar')
            if jar is not None and _OGA(self, '_Persistent__oid') is not None:
                jar.register(self)

        def _p_set_changed_flag(self, value):
            if value:
                before = _OGA(self, '_Persistent__flags')
                after = before | _CHANGED
                if before != after:
                    self._p_register()
                _OSA(self, '_Persistent__flags', after)
            else:
                flags = _OGA(self, '_Persistent__flags')
                flags &= ~_CHANGED
                _OSA(self, '_Persistent__flags', flags)

        def _p_accessed(self):
            # Notify the jar's pickle cache that we have been accessed.
            # This relies on what has been (until now) an implementation
            # detail, the '_cache' attribute of the jar.  We made it a
            # private API to avoid the cycle of keeping a reference to
            # the cache on the persistent object.

            # The below is the equivalent of this, but avoids
            # several recursive through __getattribute__, especially for _p_state,
            # and benchmarks much faster
            #
            # if(self.__jar is  None or
            #    self.__oid is None or
            #    self._p_state < 0 ): return

            oga = _OGA
            jar = oga(self, '_Persistent__jar')
            if jar is None:
                return
            oid = oga(self, '_Persistent__oid')
            if oid is None:
                return
            flags = oga(self, '_Persistent__flags')
            if flags is None: # ghost
                return

            # The KeyError arises in ZODB: ZODB.serialize.ObjectWriter
            # can assign a jar and an oid to newly seen persistent objects,
            # but because they are newly created, they aren't in the
            # pickle cache yet. There doesn't seem to be a way to distinguish
            # that at this level, all we can do is catch it.
            # The AttributeError arises in ZODB test cases
            try:
                jar._cache.mru(oid)
            except (AttributeError,KeyError):
                pass

        def _p_is_in_cache(self, jar=None):
            oid = _OGA(self, '_Persistent__oid')
            if not oid:
                return False

            jar = jar or _OGA(self, '_Persistent__jar')
            cache = getattr(jar, '_cache', None)
            if cache is not None:
                return cache.get(oid) is self

        def __repr__(self):
            p_repr_str = ''
            p_repr = getattr(type(self), '_p_repr', None)
            if p_repr is not None:
                try:
                    return p_repr(self)
                except Exception as e:
                    p_repr_str = ' _p_repr %r' % (e,)

            oid = _OGA(self, '_Persistent__oid')
            jar = _OGA(self, '_Persistent__jar')

            oid_str = ''
            jar_str = ''

            if oid is not None:
                try:
                    if isinstance(oid, bytes) and len(oid) == 8:
                        oid_str = ' oid 0x%x' % (_OID_UNPACK(oid)[0],)
                    else:
                        oid_str = ' oid %r' % (oid,)
                except Exception as e:
                    oid_str = ' oid %r' % (e,)

            if jar is not None:
                try:
                    jar_str = ' in %r' % (jar,)
                except Exception as e:
                    jar_str = ' in %r' % (e,)

            return '<%s.%s object at 0x%x%s%s%s>' % (
                # Match the C name for this exact class
                type(self).__module__ if type(self) is not Persistent else 'persistent',
                type(self).__name__, id(self),
                oid_str, jar_str, p_repr_str
            )

    def _estimated_size_in_24_bits(value):
        if value > 1073741696:
            return 16777215
        return (value//64) + 1

    _SPECIAL_NAMES.update([intern('_Persistent' + x) for x in Persistent.__slots__])