Skip to main content
`Ctrl`+`K`

# Source code for astropy.utils.masked.core

    # Licensed under a 3-clause BSD style license - see LICENSE.rst
    """
    Built-in mask mixin class.

    The design uses `Masked` as a factory class which automatically
    generates new subclasses for any data class that is itself a
    subclass of a predefined masked class, with `MaskedNDArray`
    providing such a predefined class for `~numpy.ndarray`.

    Generally, any new predefined class should override the
    ``from_unmasked(data, mask, copy=False)`` class method that
    creates an instance from unmasked data and a mask, as well as
    the ``unmasked`` property that returns just the data.
    The `Masked` class itself provides a base ``mask`` property,
    which can also be overridden if needed.

    """

    import abc
    import builtins
    import importlib

    import numpy as np

    from astropy.utils.compat import NUMPY_LT_2_5
    from astropy.utils.data_info import ParentDtypeInfo
    from astropy.utils.shapes import NDArrayShapeMethods, ShapedLikeNDArray

    from .function_helpers import (
        APPLY_TO_BOTH_FUNCTIONS,
        DISPATCHED_FUNCTIONS,
        MASKED_SAFE_FUNCTIONS,
        UNSUPPORTED_FUNCTIONS,
    )

    __all__ = [
        "MaskableShapedLikeNDArray",
        "Masked",
        "MaskedNDArray",
        "combine_masks",
        "get_data_and_mask",
    ]

    get__doc__ = """Masked version of {0.__name__}.

    Except for the ability to pass in a ``mask``, parameters are
    as for `{0.__module__}.{0.__name__}`.
    """.format

    def get_data_and_mask(array):
        """Split possibly masked array into unmasked and mask.

        Parameters
        ----------
        array : array-like
            Possibly masked item, judged by whether it has a ``mask`` attribute
            (but ignored when a possible ``masked`` attribute is `False`).
            If so, checks for having an ``unmasked`` attribute (as expected for
            instances of `~astropy.utils.masked.Masked`), or uses the ``_data``
            attribute if the inpuit is an instance of `~numpy.ma.MaskedArray`.

        Returns
        -------
        unmasked, mask : array-like
            If the input array had no mask, this will be ``array, None``.

        Raises
        ------
        AttributeError
            If ``array`` has a ``mask`` but not an ``unmasked`` attribute, and
            is not an instance of `~numpy.ma.MaskedArray`.
        ValueError
            If ``array`` is ``np.ma.masked`` (since it has no data).

        """
        mask = getattr(array, "mask", None)
        if mask is None or not getattr(array, "masked", True):
            return array, None

        try:
            return array.unmasked, mask
        except AttributeError as exc:
            if not isinstance(array, np.ma.MaskedArray):
                raise AttributeError(
                    f"'{type(array).__name__}' object has a 'mask' attribute but not an "
                    "'unmasked' attribute (and is not an np.ma.MaskedArray instance)."
                ) from None

        if array is np.ma.masked:
            raise ValueError("cannot handle np.ma.masked.")

        # We use the private _data attribute here since MaskedColumn
        # overrides the normal ".data".
        return array._data, mask

    def combine_masks(masks, *, out=None, where=True, copy=True):
        """Combine masks, possibly storing it in some output.

        Parameters
        ----------
        masks : tuple of array of bool or False or None
            Input masks.  Any that are `None` or `False` are ignored.
            Should broadcast to each other.  For structured dtype,
            an element is considered masked if any of the fields is.
        out : array, optional
            Possible output array to hold the result.
        where : array of bool, optional
            Which elements of the output array to fill.
        copy : bool optional
            Whether to ensure a copy is made. Only relevant if just a
            single input mask is not `None`, and ``out`` is not given.

        Returns
        -------
        mask : array
            Combined mask.
        """
        # Simplify masks, by removing empty ones and combining possible fields.
        masks = [
            m if m.dtype.names is None else (m != np.zeros((), dtype=m.dtype))
            for m in masks
            if m is not None and m is not False
        ]
        if not masks:
            if out is None:
                return False
            else:
                # Use copyto to deal with broadcasting with `where`.
                np.copyto(out, False, where=where)
                return out

        if len(masks) == 1:
            if out is None:
                return masks[0].copy() if copy else masks[0]
            else:
                np.copyto(out, masks[0], where=where)
                return out

        result = np.logical_or(masks[0], masks[1], out=out, where=where)
        for mask in masks[2:]:
            result = np.logical_or(result, mask, out=out, where=where)
        return result

    class Masked(NDArrayShapeMethods):
        """A scalar value or array of values with associated mask.

        The resulting instance will take its exact type from whatever the
        contents are, with the type generated on the fly as needed.

        Parameters
        ----------
        data : array-like
            The data for which a mask is to be added.  The result will be a
            a subclass of the type of ``data``.
        mask : array-like of bool, optional
            The initial mask to assign.  If not given, taken from the data.
            If the data already has a mask, the masks are combined.
        copy : bool
            Whether the data and mask should be copied. Default: `False`.

        """

        _base_classes = {}
        """Explicitly defined masked classes keyed by their unmasked counterparts.

        For subclasses of these unmasked classes, masked counterparts can be generated.
        """

        _masked_classes = {}
        """Masked classes keyed by their unmasked data counterparts."""

        def __new__(cls, *args, **kwargs):
            if cls is Masked:
                # Initializing with Masked itself means we're in "factory mode".
                if not kwargs and len(args) == 1 and isinstance(args[0], type):
                    # Create a new masked class.
                    return cls._get_masked_cls(args[0])
                else:
                    return cls._get_masked_instance(*args, **kwargs)
            else:
                # Otherwise we're a subclass and should just pass information on.
                return super().__new__(cls, *args, **kwargs)

        def __init_subclass__(cls, base_cls=None, data_cls=None, **kwargs):
            """Register a Masked subclass.

            Parameters
            ----------
            base_cls : type, optional
                If given, it is taken to mean that ``cls`` can be used as
                a base for masked versions of all subclasses of ``base_cls``,
                so it is registered as such in ``_base_classes``.
            data_cls : type, optional
                If given, ``cls`` should will be registered as the masked version of
                ``data_cls``.  Will set the private ``cls._data_cls`` attribute,
                and auto-generate a docstring if not present already.
            **kwargs
                Passed on for possible further initialization by superclasses.

            """
            if base_cls is not None:
                Masked._base_classes[base_cls] = cls

            if data_cls is not None:
                cls._data_cls = data_cls
                cls._masked_classes[data_cls] = cls
                if cls.__doc__ is None:
                    cls.__doc__ = get__doc__(data_cls)

            super().__init_subclass__(**kwargs)

        # This base implementation just uses the class initializer.
        # Subclasses can override this in case the class does not work
        # with this signature, or to provide a faster implementation.

        @classmethod
        def from_unmasked(cls, data, mask=None, copy=None):
            """Create an instance from unmasked data and a mask."""
            return cls(data, mask=mask, copy=copy)

        @classmethod
        def _get_masked_instance(cls, data, mask=None, copy=None):
            data, data_mask = get_data_and_mask(data)
            if mask is None:
                mask = False if data_mask is None else data_mask
            elif data_mask is not None:
                mask = mask | data_mask

            masked_cls = cls._get_masked_cls(data.__class__)
            return masked_cls.from_unmasked(data, mask, copy)

        @classmethod
        def _get_masked_cls(cls, data_cls):
            """Get the masked wrapper for a given data class.

            If the data class does not exist yet but is a subclass of any of the
            registered base data classes, it is automatically generated
            (except we skip `~numpy.ma.MaskedArray` subclasses, since then the
            masking mechanisms would interfere).
            """
            if issubclass(data_cls, (Masked, np.ma.MaskedArray)):
                return data_cls

            masked_cls = cls._masked_classes.get(data_cls)
            if masked_cls is None:
                # Walk through MRO and find closest base data class.
                # Note: right now, will basically always be ndarray, but
                # one could imagine needing some special care for one subclass,
                # which would then get its own entry.  E.g., if MaskedAngle
                # defined something special, then MaskedLongitude should depend
                # on it.
                for mro_item in data_cls.__mro__:
                    base_cls = cls._base_classes.get(mro_item)
                    if base_cls is not None:
                        break
                else:
                    # Just hope that MaskedNDArray can handle it.
                    # TODO: this covers the case where a user puts in a list or so,
                    # but for those one could just explicitly do something like
                    # _masked_classes[list] = MaskedNDArray.
                    return MaskedNDArray

                # Create (and therefore register) new Masked subclass for the
                # given data_cls.
                masked_cls = type(
                    "Masked" + data_cls.__name__,
                    (data_cls, base_cls),
                    {},
                    data_cls=data_cls,
                )

            return masked_cls

        def _get_mask(self):
            """The mask.

            If set, replace the original mask, with whatever it is set with,
            using a view if no broadcasting or type conversion is required.
            """
            return self._mask

        def _set_mask(self, mask, copy=False):
            self_dtype = getattr(self, "dtype", None)
            mask_dtype = (
                np.ma.make_mask_descr(self_dtype)
                if self_dtype and self_dtype.names
                else np.dtype("?")
            )
            ma = np.asanyarray(mask, dtype=mask_dtype)
            if ma.shape != self.shape:
                # This will fail (correctly) if not broadcastable.
                self._mask = np.empty(self.shape, dtype=mask_dtype)
                self._mask[...] = ma
            elif ma is mask:
                # Even if not copying use a view so that shape setting
                # does not propagate.
                self._mask = mask.copy() if copy else mask.view()
            else:
                self._mask = ma

        mask = property(_get_mask, _set_mask)

        # Note: subclass should generally override the unmasked property.
        # This one assumes the unmasked data is stored in a private attribute.
        @property
        def unmasked(self):
            """The unmasked values.

            See Also
            --------
            astropy.utils.masked.Masked.filled
            """
            return self._unmasked

        def filled(self, fill_value):
            """Get a copy of the underlying data, with masked values filled in.

            Parameters
            ----------
            fill_value : object
                Value to replace masked values with.

            See Also
            --------
            astropy.utils.masked.Masked.unmasked
            """
            unmasked = self.unmasked.copy()
            if self.mask.dtype.names:
                np.ma.core._recursive_filled(unmasked, self.mask, fill_value)
            else:
                unmasked[self.mask] = fill_value

            return unmasked

        def _apply(self, method, *args, **kwargs):
            # Required method for NDArrayShapeMethods, to help provide __getitem__
            # and shape-changing methods.
            if callable(method):
                data = method(self.unmasked, *args, **kwargs)
                mask = method(self.mask, *args, **kwargs)
            else:
                data = getattr(self.unmasked, method)(*args, **kwargs)
                mask = getattr(self.mask, method)(*args, **kwargs)

            result = self.from_unmasked(data, mask, copy=None)
            if "info" in self.__dict__:
                result.info = self.info

            return result

        def __setitem__(self, item, value):
            if value is np.ma.masked or value is np.ma.nomask:
                mask = value is np.ma.masked
            else:
                unmasked, mask = get_data_and_mask(value)
                self.unmasked[item] = unmasked
            self.mask[item] = mask

    class MaskableShapedLikeNDArray(ShapedLikeNDArray):
        """Like ShapedLikeNDArray, but for classes that can work with masked data.

        Defines default unmasked property as well as a filled method, and inherits
        private class methods that help deal with masked inputs.

        Any class using this must provide a masked property, which tells whether
        the underlying data are Masked, as well as a mask property, which
        generally should provide a read-only copy of the underlying mask.

        """

        @property
        @abc.abstractmethod
        def masked(self):
            """Whether or not the instance uses masked values."""

        @property
        @abc.abstractmethod
        def mask(self):
            """The mask."""

        @property
        def unmasked(self):
            """Get an instance without the mask.

            Note that while one gets a new instance, the underlying data will be shared.

            See Also
            --------
            filled : get a copy of the underlying data, with masked values filled in.
            """
            return self._apply(lambda x: getattr(x, "unmasked", x))

        def filled(self, fill_value):
            """Get a copy of the underlying data, with masked values filled in.

            Parameters
            ----------
            fill_value : object
                Value to replace masked values with.

            Returns
            -------
            filled : instance
                Copy of ``self`` with masked items replaced by ``fill_value``.

            See Also
            --------
            unmasked : get an instance without the mask.
            """
            unmasked = self.unmasked.copy()
            unmasked[self.mask] = fill_value
            return unmasked

    class MaskedInfoBase:
        mask_val = np.ma.masked

        def __init__(self, bound=False):
            super().__init__(bound)

            # If bound to a data object instance then create the dict of attributes
            # which stores the info attribute values.
            if bound:
                # Specify how to serialize this object depending on context.
                self.serialize_method = {
                    "fits": "null_value",
                    "ecsv": "null_value",
                    "hdf5": "data_mask",
                    "parquet": "data_mask",
                    None: "null_value",
                }

    class MaskedNDArrayInfo(MaskedInfoBase, ParentDtypeInfo):
        """
        Container for meta information like name, description, format.
        """

        # Add `serialize_method` attribute to the attrs that MaskedNDArrayInfo knows
        # about.  This allows customization of the way that MaskedColumn objects
        # get written to file depending on format.  The default is to use whatever
        # the writer would normally do, which in the case of FITS or ECSV is to use
        # a NULL value within the data itself.  If serialize_method is 'data_mask'
        # then the mask is explicitly written out as a separate column if there
        # are any masked values.  This is the same as for MaskedColumn.
        attr_names = ParentDtypeInfo.attr_names | {"serialize_method"}

        # When `serialize_method` is 'data_mask', and data and mask are being written
        # as separate columns, use column names <name> and <name>.mask (instead
        # of default encoding as <name>.data and <name>.mask).
        _represent_as_dict_primary_data = "data"

        def _represent_as_dict(self):
            out = super()._represent_as_dict()

            masked_array = self._parent

            # If the serialize method for this context (e.g. 'fits' or 'ecsv') is
            # 'data_mask', that means to serialize using an explicit mask column.
            method = self.serialize_method[self._serialize_context]

            if method == "data_mask":
                out["data"] = masked_array.unmasked

                if np.any(masked_array.mask):
                    # Only if there are actually masked elements do we add the ``mask`` column
                    out["mask"] = masked_array.mask

            elif method == "null_value":
                out["data"] = np.ma.MaskedArray(
                    masked_array.unmasked, mask=masked_array.mask
                )

            else:
                raise ValueError(
                    'serialize method must be either "data_mask" or "null_value"'
                )

            return out

        def _construct_from_dict(self, map):
            # Override usual handling, since MaskedNDArray takes shape and buffer
            # as input, which is less useful here.
            # The map can contain either a MaskedColumn or a Column and a mask.
            # Extract the mask for the former case.
            map.setdefault("mask", getattr(map["data"], "mask", False))
            return self._parent_cls.from_unmasked(**map)

    class MaskedArraySubclassInfo(MaskedInfoBase):
        """Mixin class to create a subclasses such as MaskedQuantityInfo."""

        # This is used below in __init_subclass__, which also inserts a
        # 'serialize_method' attribute in attr_names.

        def _represent_as_dict(self):
            # Use the data_cls as the class name for serialization,
            # so that we do not have to store all possible masked classes
            # in astropy.table.serialize.__construct_mixin_classes.
            out = super()._represent_as_dict()
            data_cls = self._parent._data_cls
            out.setdefault("__class__", data_cls.__module__ + "." + data_cls.__name__)
            return out

    def _comparison_method(op):
        """
        Create a comparison operator for MaskedNDArray.

        Needed since for string dtypes the base operators bypass __array_ufunc__
        and hence return unmasked results.
        """

        def _compare(self, other):
            other_data, other_mask = get_data_and_mask(other)
            result = getattr(self.unmasked, op)(other_data)
            if result is NotImplemented:
                return NotImplemented
            mask = self.mask | (other_mask if other_mask is not None else False)
            return self._masked_result(result, mask, None)

        return _compare

    class MaskedIterator:
        """
        Flat iterator object to iterate over Masked Arrays.

        A `~astropy.utils.masked.MaskedIterator` iterator is returned by ``m.flat``
        for any masked array ``m``.  It allows iterating over the array as if it
        were a 1-D array, either in a for-loop or by calling its `next` method.

        Iteration is done in C-contiguous style, with the last index varying the
        fastest. The iterator can also be indexed using basic slicing or
        advanced indexing.

        Notes
        -----
        The design of `~astropy.utils.masked.MaskedIterator` follows that of
        `~numpy.ma.core.MaskedIterator`.  It is not exported by the
        `~astropy.utils.masked` module.  Instead of instantiating directly,
        use the ``flat`` method in the masked array instance.
        """

        def __init__(self, m):
            self._masked = m
            self._dataiter = m.unmasked.flat
            self._maskiter = m.mask.flat

        def __iter__(self):
            return self

        def __getitem__(self, index):
            out = self._dataiter.__getitem__(index)
            mask = self._maskiter.__getitem__(index)
            # For single elements, ndarray.flat.__getitem__ returns scalars; these
            # need a new view as a Masked array.
            if not isinstance(out, np.ndarray):
                out = out[...]
                mask = mask[...]

            return self._masked.from_unmasked(out, mask, copy=False)

        def __setitem__(self, index, value):
            if value is np.ma.masked or value is np.ma.nomask:
                mask = value is np.ma.masked
            else:
                unmasked, mask = get_data_and_mask(value)
                self._dataiter[index] = unmasked
            self._maskiter[index] = mask

        def __next__(self):
            """
            Return the next value, or raise StopIteration.
            """
            out = next(self._dataiter)[...]
            mask = next(self._maskiter)[...]
            return self._masked.from_unmasked(out, mask, copy=False)

        next = __next__

    class MaskedNDArray(Masked, np.ndarray, base_cls=np.ndarray, data_cls=np.ndarray):
        _mask = None

        info = MaskedNDArrayInfo()

        def __new__(cls, *args, mask=None, **kwargs):
            """Get data class instance from arguments and then set mask."""
            self = super().__new__(cls, *args, **kwargs)
            if mask is not None:
                self.mask = mask
            elif self._mask is None:
                self.mask = False
            return self

        def __init_subclass__(cls, **kwargs):
            super().__init_subclass__(cls, **kwargs)
            # For all subclasses we should set a default __new__ that passes on
            # arguments other than mask to the data class, and then sets the mask.
            if "__new__" not in cls.__dict__:

                def __new__(newcls, *args, mask=None, **kwargs):
                    """Get data class instance from arguments and then set mask."""
                    # Need to explicitly mention classes outside of class definition.
                    self = super(cls, newcls).__new__(newcls, *args, **kwargs)
                    if mask is not None:
                        self.mask = mask
                    elif self._mask is None:
                        self.mask = False
                    return self

                cls.__new__ = __new__

            if "info" not in cls.__dict__ and hasattr(cls._data_cls, "info"):
                data_info = cls._data_cls.info
                attr_names = data_info.attr_names | {"serialize_method"}
                # Ensure the new info class uses the class's module.
                # Without this, the DataInfoMeta metaclass incorrectly sets
                # __module__ to its own.
                new_info = type(
                    cls.__name__ + "Info",
                    (MaskedArraySubclassInfo, data_info.__class__),
                    dict(attr_names=attr_names, __module__=cls.__module__),
                )
                cls.info = new_info()

        # The two pieces typically overridden.

        @classmethod
        def from_unmasked(cls, data, mask=None, copy=None):
            # Note: have to override since __new__ would use ndarray.__new__
            # which expects the shape as its first argument, not an array.
            data = np.array(data, subok=True, copy=copy)
            self = data.view(cls)
            self._set_mask(mask, copy=copy)
            return self

        @property
        def unmasked(self):
            return super().view(self._data_cls)

        @classmethod
        def _get_masked_cls(cls, data_cls):
            # Short-cuts
            if data_cls is np.ndarray:
                return MaskedNDArray
            elif data_cls is None:  # for .view()
                return cls

            return super()._get_masked_cls(data_cls)

        @property
        def flat(self):
            """A 1-D iterator over the Masked array.

            This returns a ``MaskedIterator`` instance, which behaves the same
            as the `~numpy.flatiter` instance returned by `~numpy.ndarray.flat`,
            and is similar to Python's built-in iterator, except that it also
            allows assignment.
            """
            return MaskedIterator(self)

        @property
        def _baseclass(self):
            """Work-around for MaskedArray initialization.

            Allows the base class to be inferred correctly when a masked instance
            is used to initialize (or viewed as) a `~numpy.ma.MaskedArray`.

            """
            return self._data_cls

        def view(self, dtype=None, type=None):
            """New view of the masked array.

            Like `numpy.ndarray.view`, but always returning a masked array subclass.
            """
            if type is None and (
                isinstance(dtype, builtins.type) and issubclass(dtype, np.ndarray)
            ):
                return super().view(self._get_masked_cls(dtype))

            if dtype is None:
                return super().view(self._get_masked_cls(type))

            dtype = np.dtype(dtype)
            result = super().view(dtype, self._get_masked_cls(type))
            # Mask should be viewed in all but simplest case.
            if (
                dtype.itemsize != self.dtype.itemsize
                or dtype.names
                or dtype.shape
                or self.dtype.names
                or self.dtype.shape
            ):
                try:
                    result.mask = self.mask.view(np.ma.make_mask_descr(dtype))
                except Exception as exc:
                    raise NotImplementedError(
                        f"{self.__class__} cannot be viewed with a dtype "
                        "with a different number of fields or size."
                    ) from None

            return result

        def __array_finalize__(self, obj):
            # If we're a new object or viewing an ndarray, nothing has to be done.
            if obj is None or obj.__class__ is np.ndarray:
                return

            # Logically, this should come from ndarray and hence do nothing, but
            # just in case someone creates a new mixin, we run it.
            super().__array_finalize__(obj)

            if self._mask is None:
                # Got here after, e.g., a view of another masked class.
                # Get its mask, or initialize ours.
                self._set_mask(getattr(obj, "_mask", False))

            if "info" in obj.__dict__:
                self.info = obj.info

        @property
        def shape(self):
            """The shape of the data and the mask.

            Usually used to get the current shape of an array, but may also be
            used to reshape the array in-place by assigning a tuple of array
            dimensions to it.  As with `numpy.reshape`, one of the new shape
            dimensions can be -1, in which case its value is inferred from the
            size of the array and the remaining dimensions.

            Raises
            ------
            AttributeError
                If a copy is required, of either the data or the mask.

            """
            # Redefinition to allow defining a setter and add a docstring.
            return super().shape

        @shape.setter
        def shape(self, shape):
            return self._set_shape(shape)

        def _set_shape(self, shape):
            old_shape = self.shape
            if NUMPY_LT_2_5:
                self._mask.shape = shape
            else:
                self._mask._set_shape(shape)
            # Reshape array proper in try/except just in case some broadcasting
            # or so causes it to fail.
            try:
                if NUMPY_LT_2_5:
                    super(MaskedNDArray, type(self)).shape.__set__(self, shape)
                else:
                    super()._set_shape(shape)
            except Exception as exc:
                if NUMPY_LT_2_5:
                    self._mask.shape = old_shape
                else:
                    self._mask._set_shape(old_shape)
                # Given that the mask reshaping succeeded, the only logical
                # reason for an exception is something like a broadcast error in
                # in __array_finalize__, or a different memory ordering between
                # mask and data.  For those, give a more useful error message;
                # otherwise just raise the error.
                if "could not broadcast" in exc.args[0]:
                    raise AttributeError(
                        "Incompatible shape for in-place modification. "
                        "Use `.reshape()` to make a copy with the desired "
                        "shape."
                    ) from None
                else:  # pragma: no cover
                    raise

        _eq_simple = _comparison_method("__eq__")
        _ne_simple = _comparison_method("__ne__")
        __lt__ = _comparison_method("__lt__")
        __le__ = _comparison_method("__le__")
        __gt__ = _comparison_method("__gt__")
        __ge__ = _comparison_method("__ge__")

        def __eq__(self, other):
            if not self.dtype.names:
                return self._eq_simple(other)

            # For structured arrays, we treat this as a reduction over the fields,
            # where masked fields are skipped and thus do not influence the result.
            other = np.asanyarray(other, dtype=self.dtype)
            result = np.stack(
                [self[field] == other[field] for field in self.dtype.names], axis=-1
            )
            return result.all(axis=-1)

        def __ne__(self, other):
            if not self.dtype.names:
                return self._ne_simple(other)

            # For structured arrays, we treat this as a reduction over the fields,
            # where masked fields are skipped and thus do not influence the result.
            other = np.asanyarray(other, dtype=self.dtype)
            result = np.stack(
                [self[field] != other[field] for field in self.dtype.names], axis=-1
            )
            return result.any(axis=-1)

        @staticmethod
        def _get_data_and_masks(arrays):
            """Extracts the data and masks from the given arrays.

            Parameters
            ----------
            arrays : iterable of array
                An iterable of arrays, possibly masked.

            Returns
            -------
            datas, masks: tuple of array
                Extracted data and mask arrays. For any input array without
                a mask, the corresponding entry in ``masks`` is `None`.
            """
            data_masks = [get_data_and_mask(array) for array in arrays]
            return tuple(zip(*data_masks))

        def __array_ufunc__(self, ufunc, method, *inputs, **kwargs):
            # Get inputs and there masks.
            unmasked, masks = self._get_data_and_masks(inputs)

            # Deal with possible outputs and their masks.
            out = kwargs.get("out")
            out_mask = None
            if out is None:
                out_masks = [None] * ufunc.nout
            else:
                out_unmasked, out_masks = self._get_data_and_masks(out)
                kwargs["out"] = out_unmasked
                for d, m in zip(out_unmasked, out_masks):
                    if m is None:
                        # TODO: allow writing to unmasked output if nothing is masked?
                        if d is not None:
                            raise TypeError("cannot write to unmasked output")
                    elif out_mask is None:
                        out_mask = m

            # TODO: where is only needed for __call__ and reduce;
            # this is very fast, but still worth separating out?
            where = kwargs.get("where", True)
            if where is True:
                where_unmasked = True
                where_mask = None
            else:
                where_unmasked, where_mask = get_data_and_mask(where)
                kwargs["where"] = where_unmasked

            # First calculate the unmasked result. This will also verify kwargs.
            # It will raise if the arguments do not know how to deal with each other.
            result = getattr(ufunc, method)(*unmasked, **kwargs)

            if ufunc.signature:
                # We're dealing with a gufunc. For now, only deal with
                # np.matmul and gufuncs for which the mask of any output always
                # depends on all core dimension values of all inputs.
                # TODO: in principle, it should be possible to generate the mask
                # purely based on the signature.
                if ufunc is np.matmul:
                    # np.matmul is tricky and its signature cannot be parsed by
                    # _parse_gufunc_signature.  But we can calculate the mask
                    # with matmul by using that nan will propagate correctly.
                    # We use float16 to minimize the memory requirements.
                    nan_masks = []
                    for a, m in zip(unmasked, masks):
                        nan_mask = np.zeros(a.shape, dtype=np.float16)
                        if m is not None:
                            nan_mask[m] = np.nan
                        nan_masks.append(nan_mask)
                    m_kwargs = {
                        k: v for k, v in kwargs.items() if k not in ("out", "where")
                    }
                    t = ufunc(*nan_masks, **m_kwargs)
                    mask = np.isnan(t, out=out_mask)

                else:
                    # Parse signature with private numpy function. Note it
                    # cannot handle spaces in tuples, so remove those.
                    in_sig, out_sig = np.lib._function_base_impl._parse_gufunc_signature(
                        ufunc.signature.replace(" ", "")
                    )
                    axes = kwargs.get("axes")
                    if axes is None:
                        # Maybe axis was given? (Note: ufunc will not take both.)
                        axes = [kwargs.get("axis")] * ufunc.nargs
                    elif len(axes) < ufunc.nargs:
                        # All outputs have no core dimensions, which means axes
                        # is not needed, but add None's for the zip below.
                        axes = axes + [None] * (ufunc.nargs - len(axes))  # not inplace!
                    keepdims = kwargs.get("keepdims", False)
                    in_masks = []
                    for sig, mask, axis in zip(in_sig, masks, axes[: ufunc.nin]):
                        if mask is not None:
                            if sig:
                                if axis is None:
                                    axis = tuple(range(-1, -1 - len(sig), -1))
                                # Input has core dimensions.  Assume that if any
                                # value in those is masked, the output will be
                                # masked too (TODO: for multiple core dimensions
                                # this may be too strong).
                                mask = np.logical_or.reduce(
                                    mask, axis=axis, keepdims=keepdims
                                )
                            in_masks.append(mask)

                    if ufunc.nout == 1 and out_sig[0] == ():
                        # Special-case where possible in-place is easy.
                        mask = combine_masks(in_masks, out=out_mask, copy=False)
                    else:
                        # Here, some masks may need expansion, so we forego in-place.
                        mask = combine_masks(in_masks, copy=False)
                        result_masks = []
                        for os, omask, axis in zip(out_sig, out_masks, axes[ufunc.nin :]):
                            if os:
                                # Output has core dimensions.  Assume all those
                                # get the same mask.
                                if axis is None:
                                    axis = tuple(range(-1, -1 - len(os), -1))
                                result_mask = np.expand_dims(mask, axis)
                            else:
                                result_mask = mask
                            if omask is not None:
                                omask[...] = result_mask
                            result_masks.append(result_mask)

                        mask = result_masks if ufunc.nout > 1 else result_masks[0]

            elif method == "__call__":
                # Regular ufunc call.
                # Combine the masks from the input, possibly selecting elements.
                mask = combine_masks(masks, out=out_mask, where=where_unmasked)
                # If relevant, also mask output elements for which where was masked.
                if where_mask is not None:
                    mask |= where_mask
                if out_mask is not None:
                    # Check for any additional explicitly given outputs.
                    for m in out_masks[1:]:
                        if m is not None and m is not out_mask:
                            m[...] = mask

            elif method == "outer":
                # Must have two inputs and one output, so also only one output mask.
                # Adjust masks as will be done for data.
                m0, m1 = masks
                if m0 is not None and m0.ndim > 0:
                    m0 = m0[(...,) + (np.newaxis,) * np.ndim(unmasked[1])]
                mask = combine_masks((m0, m1), out=out_mask)

            elif method in {"reduce", "accumulate"}:
                # Reductions like np.add.reduce (sum).
                # Treat any masked where as if the input element was masked.
                mask = combine_masks((masks[0], where_mask), copy=False)
                if mask is False and out_mask is not None:
                    if where_unmasked is True:
                        out_mask[...] = False
                    else:
                        # This is too complicated, just fall through to below.
                        mask = np.broadcast_to(False, inputs[0].shape)

                if mask is not False:
                    # By default, we simply propagate masks, since for
                    # things like np.sum, it makes no sense to do otherwise.
                    # Individual methods need to override as needed.
                    if method == "reduce":
                        axis = kwargs.get("axis")
                        keepdims = kwargs.get("keepdims", False)
                        mask = np.logical_or.reduce(
                            mask,
                            where=where_unmasked,
                            axis=axis,
                            keepdims=keepdims,
                            out=out_mask,
                        )
                        if where_unmasked is not True:
                            # Mask also whole rows in which no elements were selected;
                            # those will have been left as unmasked above.
                            mask |= ~np.logical_or.reduce(
                                where_unmasked, axis=axis, keepdims=keepdims
                            )

                    else:
                        # Accumulate
                        axis = kwargs.get("axis", 0)
                        mask = np.logical_or.accumulate(mask, axis=axis, out=out_mask)

            elif method in {"reduceat", "at"}:  # pragma: no cover
                raise NotImplementedError(
                    "masked instances cannot yet deal with 'reduceat' or 'at'."
                )

            if result is None:  # pragma: no cover
                # This happens for the "at" method.
                return result

            if out is not None and ufunc.nout == 1:
                out = out[0]
            return self._masked_result(result, mask, out)

        def __array_function__(self, function, types, args, kwargs):
            # TODO: go through functions systematically to see which ones
            # work and/or can be supported.
            if function in MASKED_SAFE_FUNCTIONS:
                return super().__array_function__(function, types, args, kwargs)

            elif function in APPLY_TO_BOTH_FUNCTIONS:
                helper = APPLY_TO_BOTH_FUNCTIONS[function]
                try:
                    helper_result = helper(*args, **kwargs)
                except NotImplementedError:
                    return self._not_implemented_or_raise(function, types)

                data_args, mask_args, kwargs, out = helper_result
                if out is not None:
                    if not isinstance(out, Masked):
                        return self._not_implemented_or_raise(function, types)
                    function(*mask_args, out=out.mask, **kwargs)
                    function(*data_args, out=out.unmasked, **kwargs)
                    return out

                mask = function(*mask_args, **kwargs)
                result = function(*data_args, **kwargs)

            elif function in DISPATCHED_FUNCTIONS:
                dispatched_function = DISPATCHED_FUNCTIONS[function]
                try:
                    dispatched_result = dispatched_function(*args, **kwargs)
                except NotImplementedError:
                    return self._not_implemented_or_raise(function, types)

                if dispatched_result is None:
                    return None

                result, mask, out = dispatched_result

            elif function in UNSUPPORTED_FUNCTIONS:
                return NotImplemented

            else:  # pragma: no cover
                # By default, just pass it through for now.
                return super().__array_function__(function, types, args, kwargs)

            if mask is None:
                return result
            else:
                return self._masked_result(result, mask, out)

        def _not_implemented_or_raise(self, function, types):
            # Our function helper or dispatcher found that the function does not
            # work with Masked.  In principle, there may be another class that
            # knows what to do with us, for which we should return NotImplemented.
            # But if there is ndarray (or a non-Masked subclass of it) around,
            # it quite likely coerces, so we should just break.
            if any(issubclass(t, np.ndarray) and not issubclass(t, Masked) for t in types):
                raise TypeError(
                    f"the MaskedNDArray implementation cannot handle {function} "
                    "with the given arguments."
                ) from None
            else:
                return NotImplemented

        def _masked_result(self, result, mask, out):
            if isinstance(result, tuple):
                if out is None:
                    out = (None,) * len(result)
                if not isinstance(mask, (list, tuple)):
                    mask = (mask,) * len(result)
                return tuple(
                    self._masked_result(result_, mask_, out_)
                    for (result_, mask_, out_) in zip(result, mask, out)
                )

            if out is None:
                # Note that we cannot count on result being the same class as
                # 'self' (e.g., comparison of quantity results in an ndarray, most
                # operations on Longitude and Latitude result in Angle or
                # Quantity), so use Masked to determine the appropriate class.
                return Masked(result, mask)

            # TODO: remove this sanity check once test cases are more complete.
            assert isinstance(out, Masked)
            # For inplace, the mask will have been set already.
            return out

        def __array_wrap__(self, obj, context=None, return_scalar=False):
            if context is None:
                # Functions like np.ediff1d call __array_wrap__ to turn the array
                # into self's subclass.
                return self.from_unmasked(*get_data_and_mask(obj))

            raise NotImplementedError(
                "__array_wrap__ should not be used with a context any more since all use "
                "should go through array_function. Please raise an issue on "
                "https://github.com/astropy/astropy"
            )

        # Below are ndarray methods that need to be overridden as masked elements
        # need to be skipped and/or an initial value needs to be set.
        def _reduce_defaults(self, kwargs, initial_func=None):
            """Get default where and initial for masked reductions.

            Generally, the default should be to skip all masked elements.  For
            reductions such as np.minimum.reduce, we also need an initial value,
            which can be determined using ``initial_func``.

            """
            if "where" not in kwargs:
                kwargs["where"] = ~self.mask
            if initial_func is not None and "initial" not in kwargs:
                kwargs["initial"] = initial_func(self.unmasked)
            return kwargs

        def trace(self, offset=0, axis1=0, axis2=1, dtype=None, out=None):
            # Unfortunately, cannot override the call to diagonal inside trace, so
            # duplicate implementation in numpy/core/src/multiarray/calculation.c.
            diagonal = self.diagonal(offset=offset, axis1=axis1, axis2=axis2)
            return diagonal.sum(-1, dtype=dtype, out=out)

        def min(self, axis=None, out=None, **kwargs):
            return super().min(
                axis=axis, out=out, **self._reduce_defaults(kwargs, np.nanmax)
            )

        def max(self, axis=None, out=None, **kwargs):
            return super().max(
                axis=axis, out=out, **self._reduce_defaults(kwargs, np.nanmin)
            )

        def ptp(self, axis=None, out=None, **kwargs):
            result = self.max(axis=axis, out=out, **kwargs)
            result -= self.min(axis=axis, **kwargs)
            return result

        def nonzero(self):
            unmasked_nonzero = self.unmasked.nonzero()
            if self.ndim >= 1:
                not_masked = ~self.mask[unmasked_nonzero]
                return tuple(u[not_masked] for u in unmasked_nonzero)
            else:
                return unmasked_nonzero if not self.mask else np.nonzero([0])

        def compress(self, condition, axis=None, out=None):
            if out is not None:
                raise NotImplementedError("cannot yet give output")
            return self._apply("compress", condition, axis=axis)

        def repeat(self, repeats, axis=None):
            return self._apply("repeat", repeats, axis=axis)

        def choose(self, choices, out=None, mode="raise"):
            # Let __array_function__ take care since choices can be masked too.
            return np.choose(self, choices, out=out, mode=mode)

        def argmin(self, axis=None, out=None, *, keepdims=False):
            # TODO: should this return a masked integer array, with masks
            # if all elements were masked?
            at_min = self == self.min(axis=axis, keepdims=True)
            return at_min.filled(False).argmax(axis=axis, out=out, keepdims=keepdims)

        def argmax(self, axis=None, out=None, *, keepdims=False):
            at_max = self == self.max(axis=axis, keepdims=True)
            return at_max.filled(False).argmax(axis=axis, out=out, keepdims=keepdims)

        def argsort(self, axis=-1, kind=None, order=None, *, stable=None):
            """Returns the indices that would sort an array.

            Perform an indirect sort along the given axis on both the array
            and the mask, with masked items being sorted to the end.

            Parameters
            ----------
            axis : int or None, optional
                Axis along which to sort.  The default is -1 (the last axis).
                If None, the flattened array is used.
            kind : str or None, ignored.
                The kind of sort.  Present only to allow subclasses to work.
            order : str or list of str.
                For an array with fields defined, the fields to compare first,
                second, etc.  A single field can be specified as a string, and not
                all fields need be specified, but unspecified fields will still be
                used, in dtype order, to break ties.
            stable: bool, keyword-only, ignored
                Sort stability. Present only to allow subclasses to work.

            Returns
            -------
            index_array : ndarray, int
                Array of indices that sorts along the specified ``axis``.  Use
                ``np.take_along_axis(self, index_array, axis=axis)`` to obtain
                the sorted array.

            """
            if axis is None:
                data = self.ravel()
                axis = -1
            else:
                data = self

            if self.dtype.names:
                # As done inside the argsort implementation in multiarray/methods.c.
                if order is None:
                    order = self.dtype.names
                else:
                    order = np._core._internal._newnames(self.dtype, order)

                keys = tuple(data[name] for name in order[::-1])

            elif order is not None:
                raise ValueError("Cannot specify order when the array has no fields.")

            else:
                keys = (data,)

            return np.lexsort(keys, axis=axis)

        def sort(self, axis=-1, kind=None, order=None, *, stable=False):
            """Sort an array in-place. Refer to `numpy.sort` for full documentation.

            Notes
            -----
            Masked items will be sorted to the end. The implementation
            is via `numpy.lexsort` and thus ignores the ``kind`` and ``stable`` arguments;
            they are present only so that subclasses can pass them on.
            """
            # TODO: probably possible to do this faster than going through argsort!
            argsort_kwargs = dict(kind=kind, order=order, stable=stable)
            indices = self.argsort(axis, **argsort_kwargs)
            self[:] = np.take_along_axis(self, indices, axis=axis)

        def argpartition(self, kth, axis=-1, kind="introselect", order=None):
            # TODO: should be possible to do this faster than with a full argsort!
            return self.argsort(axis=axis, order=order)

        def partition(self, kth, axis=-1, kind="introselect", order=None):
            # TODO: should be possible to do this faster than with a full argsort!
            return self.sort(axis=axis, order=None)

        def cumsum(self, axis=None, dtype=None, out=None):
            if axis is None:
                self = self.ravel()
                axis = 0
            return np.add.accumulate(self, axis=axis, dtype=dtype, out=out)

        def cumprod(self, axis=None, dtype=None, out=None):
            if axis is None:
                self = self.ravel()
                axis = 0
            return np.multiply.accumulate(self, axis=axis, dtype=dtype, out=out)

        def clip(self, min=None, max=None, out=None, **kwargs):
            """Return an array whose values are limited to ``[min, max]``.

            Like `~numpy.clip`, but any masked values in ``min`` and ``max``
            are ignored for clipping.  The mask of the input array is propagated.
            """
            # TODO: implement this at the ufunc level.
            dmin, mmin = get_data_and_mask(min)
            dmax, mmax = get_data_and_mask(max)
            if mmin is None and mmax is None:
                # Fast path for unmasked max, min.
                return super().clip(min, max, out=out, **kwargs)

            masked_out = np.positive(self, out=out)
            out = masked_out.unmasked
            if dmin is not None:
                np.maximum(out, dmin, out=out, where=True if mmin is None else ~mmin)
            if dmax is not None:
                np.minimum(out, dmax, out=out, where=True if mmax is None else ~mmax)
            return masked_out

        def mean(self, axis=None, dtype=None, out=None, keepdims=False, *, where=True):
            # Implementation based on that in numpy/core/_methods.py
            # Cast bool, unsigned int, and int to float64 by default,
            # and do float16 at higher precision.
            is_float16_result = False
            if dtype is None:
                if issubclass(self.dtype.type, (np.integer, np.bool_)):
                    dtype = np.dtype("f8")
                elif issubclass(self.dtype.type, np.float16):
                    dtype = np.dtype("f4")
                    is_float16_result = out is None

            where = ~self.mask & where

            result = self.sum(
                axis=axis, dtype=dtype, out=out, keepdims=keepdims, where=where
            )
            n = np.add.reduce(where, axis=axis, keepdims=keepdims)

            # catch the case when an axis is fully masked to prevent div by zero:
            neq0 = n == 0
            n += neq0
            result /= n

            # correct fully-masked slice results to what is expected for 0/0 division
            result.unmasked[neq0] = np.nan

            if is_float16_result:
                result = result.astype(self.dtype)
            return result

        def var(
            self, axis=None, dtype=None, out=None, ddof=0, keepdims=False, *, where=True
        ):
            where_final = ~self.mask & where

            # Simplified implementation based on that in numpy/core/_methods.py
            n = np.add.reduce(where_final, axis=axis, keepdims=keepdims)[...]

            # Cast bool, unsigned int, and int to float64 by default.
            if dtype is None and issubclass(self.dtype.type, (np.integer, np.bool_)):
                dtype = np.dtype("f8")
            mean = self.mean(axis=axis, dtype=dtype, keepdims=True, where=where)

            x = self - mean
            x *= x.conjugate()  # Conjugate just returns x if not complex.

            result = x.sum(
                axis=axis, dtype=dtype, out=out, keepdims=keepdims, where=where_final
            )
            n -= ddof
            n = np.maximum(n, 0, out=n)
            result /= n
            result._mask |= n == 0
            return result

        def std(
            self, axis=None, dtype=None, out=None, ddof=0, keepdims=False, *, where=True
        ):
            result = self.var(
                axis=axis, dtype=dtype, out=out, ddof=ddof, keepdims=keepdims, where=where
            )
            return np.sqrt(result, out=result)

        def __bool__(self):
            # First get result from array itself; this will error if not a scalar.
            result = super().__bool__()
            return result and not self.mask

        def any(self, axis=None, out=None, keepdims=False, *, where=True):
            return np.logical_or.reduce(
                self, axis=axis, out=out, keepdims=keepdims, where=~self.mask & where
            )

        def all(self, axis=None, out=None, keepdims=False, *, where=True):
            return np.logical_and.reduce(
                self, axis=axis, out=out, keepdims=keepdims, where=~self.mask & where
            )

        # Following overrides needed since somehow the ndarray implementation
        # does not actually call these.
        def __str__(self):
            return np.array_str(self)

        def __repr__(self):
            return np.array_repr(self)

        def __format__(self, format_spec):
            string = super().__format__(format_spec)
            if self.shape == () and self.mask:
                n = min(3, max(1, len(string)))
                return " " * (len(string) - n) + "\u2014" * n
            else:
                return string

        def __hash__(self):
            # Try to be somewhat like a numpy array scalar if possible.
            if self.ndim == 0 and not self.mask:
                return hash(self.unmasked[()])

            # Will raise regular ndarray error.
            return hash((self.unmasked, self.mask))

    class MaskedRecarrayInfo(MaskedNDArrayInfo):
        # Ensure that we output a plain MaskedArray, not a masked_recarray.
        def _represent_as_dict(self):
            masked_ndarray = self._parent.view(np.ndarray)
            return masked_ndarray.info._represent_as_dict()

    class MaskedRecarray(np.recarray, MaskedNDArray, data_cls=np.recarray):
        # Explicit definition since we need to override some methods.

        info = MaskedRecarrayInfo()

        def __array_finalize__(self, obj):
            # recarray.__array_finalize__ does not do super, so we do it
            # explicitly.
            super().__array_finalize__(obj)
            super(np.recarray, self).__array_finalize__(obj)

        # __getattribute__, __setattr__, and field use these somewhat
        # obscrure ndarray methods.  TODO: override in MaskedNDArray?
        def getfield(self, dtype, offset=0):
            for field, info in self.dtype.fields.items():
                if offset == info[1] and dtype == info[0]:
                    return self[field]

            raise NotImplementedError("can only get existing field from structured dtype.")

        def setfield(self, val, dtype, offset=0):
            for field, info in self.dtype.fields.items():
                if offset == info[1] and dtype == info[0]:
                    self[field] = val
                    return

            raise NotImplementedError("can only set existing field from structured dtype.")

        def __repr__(self):
            cls_name = type(self).__name__
            out = super().__repr__().splitlines()
            prefix, _, rest = out[0].partition("(")
            out0 = cls_name + "(" + rest
            extra_space = (len(cls_name) - len(prefix)) * " "
            return "\n".join([out0] + [extra_space + o for o in out[1:]])

    def __getattr__(key):
        """Make commonly used Masked subclasses and their info classes importable for ASDF support.

        Registered types associated with ASDF converters must be importable by
        their fully qualified name. Masked classes are dynamically created and have
        apparent names like ``astropy.utils.masked.core.MaskedQuantity`` although
        they aren't actually attributes of this module. Customize module attribute
        lookup so that certain commonly used Masked classes are importable. The same
        is done for their dynamically created info classes.

        See:
        - https://asdf.readthedocs.io/en/latest/asdf/extending/converters.html#entry-point-performance-considerations
        - https://github.com/astropy/asdf-astropy/pull/253
        """
        if key.startswith(Masked.__name__):
            # TODO: avoid using a private attribute from table.
            # Can we make this more beautiful?
            from astropy.table.serialize import __construct_mixin_classes

            base_class_name = key[len(Masked.__name__) :]
            for base_class_qualname in __construct_mixin_classes:
                module, _, name = base_class_qualname.rpartition(".")
                is_info = name + "Info" == base_class_name
                if name == base_class_name or is_info:
                    base_class = getattr(importlib.import_module(module), name)
                    # Try creating the masked class
                    masked_class = Masked(base_class)
                    # But only return it if it is a standard one, not one
                    # where we just used the ndarray fallback.
                    if base_class in Masked._masked_classes:
                        return masked_class.info.__class__ if is_info else masked_class

        raise AttributeError(f"module '{__name__}' has no attribute '{key}'")