# glance.common.utils

# Source code for glance.common.utils

    # Copyright 2010 United States Government as represented by the
    # Administrator of the National Aeronautics and Space Administration.
    # Copyright 2014 SoftLayer Technologies, Inc.
    # Copyright 2015 Mirantis, Inc
    # All Rights Reserved.
    #
    #    Licensed under the Apache License, Version 2.0 (the "License"); you may
    #    not use this file except in compliance with the License. You may obtain
    #    a copy of the License at
    #
    #         http://www.apache.org/licenses/LICENSE-2.0
    #
    #    Unless required by applicable law or agreed to in writing, software
    #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
    #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
    #    License for the specific language governing permissions and limitations
    #    under the License.

    """
    System-level utilities and helper functions.
    """

    import errno
    import ipaddress

    try:
        from eventlet import sleep
    except ImportError:
        from time import sleep
    from eventlet.green import socket

    import functools
    import os
    import re
    import urllib

    import glance_store
    from oslo_config import cfg
    from oslo_log import log as logging
    from oslo_utils import excutils
    from oslo_utils import netutils
    from oslo_utils import strutils
    from webob import exc

    from glance.common import exception
    from glance.common import timeutils
    from glance.common import wsgi
    from glance.i18n import _, _LE

    CONF = cfg.CONF

    LOG = logging.getLogger(__name__)

    # Whitelist of v1 API headers of form x-image-meta-xxx
    IMAGE_META_HEADERS = ['x-image-meta-location', 'x-image-meta-size',
                          'x-image-meta-is_public', 'x-image-meta-disk_format',
                          'x-image-meta-container_format', 'x-image-meta-name',
                          'x-image-meta-status', 'x-image-meta-copy_from',
                          'x-image-meta-uri', 'x-image-meta-checksum',
                          'x-image-meta-created_at', 'x-image-meta-updated_at',
                          'x-image-meta-deleted_at', 'x-image-meta-min_ram',
                          'x-image-meta-min_disk', 'x-image-meta-owner',
                          'x-image-meta-store', 'x-image-meta-id',
                          'x-image-meta-protected', 'x-image-meta-deleted',
                          'x-image-meta-virtual_size']

    GLANCE_TEST_SOCKET_FD_STR = 'GLANCE_TEST_SOCKET_FD'

    def chunkreadable(iter, chunk_size=65536):
        """
        Wrap a readable iterator with a reader yielding chunks of
        a preferred size, otherwise leave iterator unchanged.

        :param iter: an iter which may also be readable
        :param chunk_size: maximum size of chunk
        """
        return chunkiter(iter, chunk_size) if hasattr(iter, 'read') else iter

    def chunkiter(fp, chunk_size=65536):
        """
        Return an iterator to a file-like obj which yields fixed size chunks

        :param fp: a file-like object
        :param chunk_size: maximum size of chunk
        """
        while True:
            chunk = fp.read(chunk_size)
            if chunk:
                yield chunk
            else:
                break

    def cooperative_iter(iter):
        """
        Return an iterator which schedules after each
        iteration. This can prevent eventlet thread starvation.

        :param iter: an iterator to wrap
        """
        try:
            for chunk in iter:
                sleep(0)
                yield chunk
        except Exception as err:
            with excutils.save_and_reraise_exception():
                msg = _LE("Error: cooperative_iter exception %s") % err
                LOG.error(msg)

    def cooperative_read(fd):
        """
        Wrap a file descriptor's read with a partial function which schedules
        after each read. This can prevent eventlet thread starvation.

        :param fd: a file descriptor to wrap
        """
        def readfn(*args):
            result = fd.read(*args)
            sleep(0)
            return result
        return readfn

    MAX_COOP_READER_BUFFER_SIZE = 134217728  # 128M seems like a sane buffer limit

    CONF.import_group('import_filtering_opts',
                      'glance.async_.flows._internal_plugins')

    def normalize_hostname(host):
        """Normalize IP address to standard format or return hostname.

        Uses ipaddress module to validate and normalize IP addresses, rejecting
        encoded formats. For hostnames, requires DNS resolution to ensure they
        are valid and not encoded IP attempts.

        :param host: hostname or IP address
        :returns: normalized IP address, hostname unchanged, or None
        """
        if not host:
            return host

        # NOTE(abhishekk): Try to parse as IPv4. ipaddress module only accepts
        # standard format like 127.0.0.1. It rejects encoded formats like
        # decimal (2130706433), hex (0x7f000001), or octal (017700000001).
        try:
            return str(ipaddress.IPv4Address(host))
        except ValueError:
            pass

        # NOTE(abhishekk): Try to parse as IPv6. ipaddress module only accepts
        # standard IPv6 format and rejects encoded formats.
        try:
            return str(ipaddress.IPv6Address(host))
        except ValueError:
            pass

        # NOTE(abhishekk): Not valid IP address, check as hostname. Reject pure
        # numeric strings like "2130706433" (decimal encoded IP). ipaddress module
        # rejected it, but OS might still resolve using inet_aton() if not blocked.
        if host.isdigit():
            return None

        # NOTE(abhishekk): Reject all numeric strings with dots like "127.1" or
        # "10.1". These are shorthand IP addresses. ipaddress module rejects them
        # because they need 4 octets, but OS may still resolve them. We block to
        # prevent SSRF bypass attacks.
        if all(c.isdigit() or c == '.' for c in host):
            return None

        # NOTE(abhishekk): Add trailing dot to force DNS lookup instead of numeric
        # parsing. This blocks encoded IP formats like 0x7f000001 or 127.0x0.0.1
        # because they fail DNS lookup. Only real hostnames that resolve via DNS
        # are allowed.
        testhost = host
        if not testhost.endswith('.'):
            testhost += '.'

        try:
            socket.getaddrinfo(testhost, 80)
        except socket.gaierror:
            # NOTE(abhishekk): DNS resolution failed, reject the hostname
            return None

        # NOTE(abhishekk): Valid and resolvable hostname, return unchanged
        return host

    def validate_import_uri(uri):
        """Validate requested uri for Image Import web-download.

        :param uri: target uri to be validated
        """
        if not uri:
            return False

        parsed_uri = urllib.parse.urlparse(uri)
        scheme = parsed_uri.scheme
        host = parsed_uri.hostname
        port = parsed_uri.port
        wl_schemes = CONF.import_filtering_opts.allowed_schemes
        bl_schemes = CONF.import_filtering_opts.disallowed_schemes
        wl_hosts = CONF.import_filtering_opts.allowed_hosts
        bl_hosts = CONF.import_filtering_opts.disallowed_hosts
        wl_ports = CONF.import_filtering_opts.allowed_ports
        bl_ports = CONF.import_filtering_opts.disallowed_ports

        # NOTE(jokke): Checking if both allowed and disallowed are defined and
        # logging it to inform only allowed will be obeyed.
        if wl_schemes and bl_schemes:
            bl_schemes = []
            LOG.debug("Both allowed and disallowed schemes has been configured. "
                      "Will only process allowed list.")
        if wl_hosts and bl_hosts:
            bl_hosts = []
            LOG.debug("Both allowed and disallowed hosts has been configured. "
                      "Will only process allowed list.")
        if wl_ports and bl_ports:
            bl_ports = []
            LOG.debug("Both allowed and disallowed ports has been configured. "
                      "Will only process allowed list.")

        if not scheme or ((wl_schemes and scheme not in wl_schemes) or
                          parsed_uri.scheme in bl_schemes):
            return False

        normalized_host = normalize_hostname(host)

        if not normalized_host or (
                (wl_hosts and normalized_host not in wl_hosts) or
                normalized_host in bl_hosts):
            return False

        if port and ((wl_ports and port not in wl_ports) or
                     port in bl_ports):
            return False

        return True

    class CooperativeReader(object):
        """
        An eventlet thread friendly class for reading in image data.

        When accessing data either through the iterator or the read method
        we perform a sleep to allow a co-operative yield. When there is more than
        one image being uploaded/downloaded this prevents eventlet thread
        starvation, ie allows all threads to be scheduled periodically rather than
        having the same thread be continuously active.
        """
        def __init__(self, fd):
            """
            :param fd: Underlying image file object
            """
            self.fd = fd
            self.iterator = None
            # NOTE(markwash): if the underlying supports read(), overwrite the
            # default iterator-based implementation with cooperative_read which
            # is more straightforward
            if hasattr(fd, 'read'):
                self.read = cooperative_read(fd)
            else:
                self.iterator = None
                self.buffer = b''
                self.position = 0

        def read(self, length=None):
            """Return the requested amount of bytes, fetching the next chunk of
            the underlying iterator when needed.

            This is replaced with cooperative_read in __init__ if the underlying
            fd already supports read().
            """
            if length is None:
                if len(self.buffer) - self.position > 0:
                    # if no length specified but some data exists in buffer,
                    # return that data and clear the buffer
                    result = self.buffer[self.position:]
                    self.buffer = b''
                    self.position = 0
                    return bytes(result)
                else:
                    # otherwise read the next chunk from the underlying iterator
                    # and return it as a whole. Reset the buffer, as subsequent
                    # calls may specify the length
                    try:
                        if self.iterator is None:
                            self.iterator = self.__iter__()
                        return next(self.iterator)
                    except StopIteration:
                        return b''
                    finally:
                        self.buffer = b''
                        self.position = 0
            else:
                result = bytearray()
                while len(result) < length:
                    if self.position < len(self.buffer):
                        to_read = length - len(result)
                        chunk = self.buffer[self.position:self.position + to_read]
                        result.extend(chunk)

                        # This check is here to prevent potential OOM issues if
                        # this code is called with unreasonably high values of read
                        # size. Currently it is only called from the HTTP clients
                        # of Glance backend stores, which use httplib for data
                        # streaming, which has readsize hardcoded to 8K, so this
                        # check should never fire. Regardless it still worths to
                        # make the check, as the code may be reused somewhere else.
                        if len(result) >= MAX_COOP_READER_BUFFER_SIZE:
                            raise exception.LimitExceeded()
                        self.position += len(chunk)
                    else:
                        try:
                            if self.iterator is None:
                                self.iterator = self.__iter__()
                            self.buffer = next(self.iterator)
                            self.position = 0
                        except StopIteration:
                            self.buffer = b''
                            self.position = 0
                            return bytes(result)
                return bytes(result)

        def __iter__(self):
            return cooperative_iter(self.fd.__iter__())

    class LimitingReader(object):
        """
        Reader designed to fail when reading image data past the configured
        allowable amount.
        """
        def __init__(self, data, limit,
                     exception_class=exception.ImageSizeLimitExceeded):
            """
            :param data: Underlying image data object
            :param limit: maximum number of bytes the reader should allow
            :param exception_class: Type of exception to be raised
            """
            self.data = data
            self.limit = limit
            self.bytes_read = 0
            self.exception_class = exception_class

        def __iter__(self):
            for chunk in self.data:
                self.bytes_read += len(chunk)
                if self.bytes_read > self.limit:
                    raise self.exception_class()
                else:
                    yield chunk

        def read(self, i):
            result = self.data.read(i)
            self.bytes_read += len(result)
            if self.bytes_read > self.limit:
                raise self.exception_class()
            return result

    def image_meta_to_http_headers(image_meta):
        """
        Returns a set of image metadata into a dict
        of HTTP headers that can be fed to either a Webob
        Request object or an httplib.HTTP(S)Connection object

        :param image_meta: Mapping of image metadata
        """
        headers = {}
        for k, v in image_meta.items():
            if v is not None:
                if k == 'properties':
                    for pk, pv in v.items():
                        if pv is not None:
                            headers["x-image-meta-property-%s"
                                    % pk.lower()] = str(pv)
                else:
                    headers["x-image-meta-%s" % k.lower()] = str(v)
        return headers

    def get_image_meta_from_headers(response):
        """
        Processes HTTP headers from a supplied response that
        match the x-image-meta and x-image-meta-property and
        returns a mapping of image metadata and properties

        :param response: Response to process
        """
        result = {}
        properties = {}

        if hasattr(response, 'getheaders'):  # httplib.HTTPResponse
            headers = response.getheaders()
        else:  # webob.Response
            headers = response.headers.items()

        for key, value in headers:
            key = str(key.lower())
            if key.startswith('x-image-meta-property-'):
                field_name = key[len('x-image-meta-property-'):].replace('-', '_')
                properties[field_name] = value or None
            elif key.startswith('x-image-meta-'):
                field_name = key[len('x-image-meta-'):].replace('-', '_')
                if 'x-image-meta-' + field_name not in IMAGE_META_HEADERS:
                    msg = _("Bad header: %(header_name)s") % {'header_name': key}
                    raise exc.HTTPBadRequest(msg, content_type="text/plain")
                result[field_name] = value or None
        result['properties'] = properties

        for key, nullable in [('size', False), ('min_disk', False),
                              ('min_ram', False), ('virtual_size', True)]:
            if key in result:
                try:
                    result[key] = int(result[key])
                except ValueError:
                    if nullable and result[key] == str(None):
                        result[key] = None
                    else:
                        extra = (_("Cannot convert image %(key)s '%(value)s' "
                                   "to an integer.")
                                 % {'key': key, 'value': result[key]})
                        raise exception.InvalidParameterValue(value=result[key],
                                                              param=key,
                                                              extra_msg=extra)
                if result[key] is not None and result[key] < 0:
                    extra = _('Cannot be a negative value.')
                    raise exception.InvalidParameterValue(value=result[key],
                                                          param=key,
                                                          extra_msg=extra)

        for key in ('is_public', 'deleted', 'protected'):
            if key in result:
                result[key] = strutils.bool_from_string(result[key])
        return result

    def create_mashup_dict(image_meta):
        """
        Returns a dictionary-like mashup of the image core properties
        and the image custom properties from given image metadata.

        :param image_meta: metadata of image with core and custom properties
        """

        d = {}
        for key, value in image_meta.items():
            if isinstance(value, dict):
                for subkey, subvalue in create_mashup_dict(value).items():
                    if subkey not in image_meta:
                        d[subkey] = subvalue
            else:
                d[key] = value

        return d

    def safe_mkdirs(path):
        try:
            os.makedirs(path)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

    def mutating(func):
        """Decorator to enforce read-only logic"""
        @functools.wraps(func)
        def wrapped(self, req, *args, **kwargs):
            if req.context.read_only:
                msg = "Read-only access"
                LOG.debug(msg)
                raise exc.HTTPForbidden(msg, request=req,
                                        content_type="text/plain")
            return func(self, req, *args, **kwargs)
        return wrapped

    def setup_remote_pydev_debug(host, port):
        error_msg = _LE('Error setting up the debug environment. Verify that the'
                        ' option pydev_worker_debug_host is pointing to a valid '
                        'hostname or IP on which a pydev server is listening on'
                        ' the port indicated by pydev_worker_debug_port.')

        try:
            try:
                from pydev import pydevd
            except ImportError:
                import pydevd

            pydevd.settrace(host,
                            port=port,
                            stdoutToServer=True,
                            stderrToServer=True)
            return True
        except Exception:
            with excutils.save_and_reraise_exception():
                LOG.exception(error_msg)

    def get_test_suite_socket():
        global GLANCE_TEST_SOCKET_FD_STR
        if GLANCE_TEST_SOCKET_FD_STR in os.environ:
            fd = int(os.environ[GLANCE_TEST_SOCKET_FD_STR])
            sock = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
            sock.listen(CONF.backlog)
            del os.environ[GLANCE_TEST_SOCKET_FD_STR]
            os.close(fd)
            return sock
        return None

    def is_valid_hostname(hostname):
        """Verify whether a hostname (not an FQDN) is valid."""
        return re.match('^[a-zA-Z0-9-]+$', hostname) is not None

    def is_valid_fqdn(fqdn):
        """Verify whether a host is a valid FQDN."""
        return re.match(r'^[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$', fqdn) is not None

    def parse_valid_host_port(host_port):
        """
        Given a "host:port" string, attempts to parse it as intelligently as
        possible to determine if it is valid. This includes IPv6 [host]:port form,
        IPv4 ip:port form, and hostname:port or fqdn:port form.

        Invalid inputs will raise a ValueError, while valid inputs will return
        a (host, port) tuple where the port will always be of type int.
        """

        try:
            try:
                host, port = netutils.parse_host_port(host_port)
            except Exception:
                raise ValueError(_('Host and port "%s" is not valid.') % host_port)

            if not netutils.is_valid_port(port):
                raise ValueError(_('Port "%s" is not valid.') % port)

            # First check for valid IPv6 and IPv4 addresses, then a generic
            # hostname. Failing those, if the host includes a period, then this
            # should pass a very generic FQDN check. The FQDN check for letters at
            # the tail end will weed out any hilariously absurd IPv4 addresses.

            if not (netutils.is_valid_ipv6(host) or netutils.is_valid_ipv4(host) or
                    is_valid_hostname(host) or is_valid_fqdn(host)):
                raise ValueError(_('Host "%s" is not valid.') % host)

        except Exception as ex:
            raise ValueError(_('%s '
                               'Please specify a host:port pair, where host is an '
                               'IPv4 address, IPv6 address, hostname, or FQDN. If '
                               'using an IPv6 address, enclose it in brackets '
                               'separately from the port (i.e., '
                               '"[fe80::a:b:c]:9876").') % ex)

        return (host, int(port))

    try:
        REGEX_4BYTE_UNICODE = re.compile(u'[\U00010000-\U0010ffff]')
    except re.error:
        # UCS-2 build case
        REGEX_4BYTE_UNICODE = re.compile(u'[\uD800-\uDBFF][\uDC00-\uDFFF]')

    def no_4byte_params(f):
        """
        Checks that no 4 byte unicode characters are allowed
        in dicts' keys/values and string's parameters
        """
        def wrapper(*args, **kwargs):

            def _is_match(some_str):
                return (
                    isinstance(some_str, str) and
                    REGEX_4BYTE_UNICODE.findall(some_str) != []
                )

            def _check_dict(data_dict):
                # a dict of dicts has to be checked recursively
                for key, value in data_dict.items():
                    if isinstance(value, dict):
                        _check_dict(value)
                    else:
                        if _is_match(key):
                            msg = _("Property names can't contain 4 byte unicode.")
                            raise exception.Invalid(msg)
                        if _is_match(value):
                            msg = (_("%s can't contain 4 byte unicode characters.")
                                   % key.title())
                            raise exception.Invalid(msg)

            for data_dict in [arg for arg in args if isinstance(arg, dict)]:
                _check_dict(data_dict)
            # now check args for str values
            for arg in args:
                if _is_match(arg):
                    msg = _("Param values can't contain 4 byte unicode.")
                    raise exception.Invalid(msg)
            # check kwargs as well, as params are passed as kwargs via
            # registry calls
            _check_dict(kwargs)
            return f(*args, **kwargs)
        return wrapper

    def stash_conf_values():
        """
        Make a copy of some of the current global CONF's settings.
        Allows determining if any of these values have changed
        when the config is reloaded.
        """
        conf = {
            'bind_host': CONF.bind_host,
            'bind_port': CONF.bind_port,
            'backlog': CONF.backlog,
        }

        return conf

    def split_filter_op(expression):
        """Split operator from threshold in an expression.
        Designed for use on a comparative-filtering query field.
        When no operator is found, default to an equality comparison.

        :param expression: the expression to parse

        :returns: a tuple (operator, threshold) parsed from expression
        """
        left, sep, right = expression.partition(':')
        if sep:
            # If the expression is a date of the format ISO 8601 like
            # CCYY-MM-DDThh:mm:ss+hh:mm and has no operator, it should
            # not be partitioned, and a default operator of eq should be
            # assumed.
            try:
                timeutils.parse_isotime(expression)
                op = 'eq'
                threshold = expression
            except ValueError:
                op = left
                threshold = right
        else:
            op = 'eq'  # default operator
            threshold = left

        # NOTE stevelle decoding escaped values may be needed later
        return op, threshold

    def validate_quotes(value):
        """Validate filter values

        Validation opening/closing quotes in the expression.
        """
        open_quotes = True
        for i in range(len(value)):
            if value[i] == '"':
                if i and value[i - 1] == '\\':
                    continue
                if open_quotes:
                    if i and value[i - 1] != ',':
                        msg = _("Invalid filter value %s. There is no comma "
                                "before opening quotation mark.") % value
                        raise exception.InvalidParameterValue(message=msg)
                else:
                    if i + 1 != len(value) and value[i + 1] != ",":
                        msg = _("Invalid filter value %s. There is no comma "
                                "after closing quotation mark.") % value
                        raise exception.InvalidParameterValue(message=msg)
                open_quotes = not open_quotes
        if not open_quotes:
            msg = _("Invalid filter value %s. The quote is not closed.") % value
            raise exception.InvalidParameterValue(message=msg)

    def split_filter_value_for_quotes(value):
        """Split filter values

        Split values by commas and quotes for 'in' operator, according api-wg.
        """
        validate_quotes(value)
        tmp = re.compile(r'''
            "(                 # if found a double-quote
               [^\"\\]*        # take characters either non-quotes or backslashes
               (?:\\.          # take backslashes and character after it
                [^\"\\]*)*     # take characters either non-quotes or backslashes
             )                 # before double-quote
            ",?                # a double-quote with comma maybe
            | ([^,]+),?        # if not found double-quote take any non-comma
                               # characters with comma maybe
            | ,                # if we have only comma take empty string
            ''', re.VERBOSE)
        return [val[0] or val[1] for val in re.findall(tmp, value)]

    def evaluate_filter_op(value, operator, threshold):
        """Evaluate a comparison operator.
        Designed for use on a comparative-filtering query field.

        :param value: evaluated against the operator, as left side of expression
        :param operator: any supported filter operation
        :param threshold: to compare value against, as right side of expression

        :raises InvalidFilterOperatorValue: if an unknown operator is provided

        :returns: boolean result of applied comparison

        """
        if operator == 'gt':
            return value > threshold
        elif operator == 'gte':
            return value >= threshold
        elif operator == 'lt':
            return value < threshold
        elif operator == 'lte':
            return value <= threshold
        elif operator == 'neq':
            return value != threshold
        elif operator == 'eq':
            return value == threshold

        msg = _("Unable to filter on a unknown operator.")
        raise exception.InvalidFilterOperatorValue(msg)

    def _get_available_stores():
        available_stores = CONF.enabled_backends
        stores = []
        # Remove reserved stores from the available stores list
        for store in available_stores:
            # NOTE (abhishekk): http store is readonly and should be
            # excluded from the list.
            if available_stores[store] == 'http':
                continue
            if store not in wsgi.RESERVED_STORES:
                stores.append(store)

        return stores

    def get_stores_from_request(req, body):
        """Processes a supplied request and extract stores from it

        :param req: request to process
        :param body: request body

        :raises glance_store.UnknownScheme:  if a store is not valid
        :return: a list of stores
        """
        if body.get('all_stores', False):
            if 'stores' in body or 'x-image-meta-store' in req.headers:
                msg = _("All_stores parameter can't be used with "
                        "x-image-meta-store header or stores parameter")
                raise exc.HTTPBadRequest(explanation=msg)
            stores = _get_available_stores()
        else:
            try:
                stores = body['stores']
            except KeyError:
                stores = [req.headers.get('x-image-meta-store',
                                          CONF.glance_store.default_backend)]
            else:
                if 'x-image-meta-store' in req.headers:
                    msg = _("Stores parameter and x-image-meta-store header can't "
                            "be both specified")
                    raise exc.HTTPBadRequest(explanation=msg)
        # Validate each store
        for store in stores:
            glance_store.get_store_from_store_identifier(store)
        return stores

this page last updated: 2026-03-19 02:17:34

Except where otherwise noted, this document is licensed under Creative Commons Attribution 3.0 License. See all OpenStack Legal Documents.

found an error? report a bug