Skip to content

# `mistral_common.tokens.tokenizers.utils`

## `chunks(lst, chunk_size)`

Chunk a list into smaller lists of a given size.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`lst` | `list[str]` | The list to chunk. | required
`chunk_size` | `int` | The size of each chunk. | required

Returns:

Type | Description
--- | ---
`Iterator[list[str]]` | An iterator over the chunks.

Examples:

    >>> all_chunks = list(chunks([1, 2, 3, 4, 5], 2))

Source code in `src/mistral_common/tokens/tokenizers/utils.py`

    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46

    def chunks(lst: list[str], chunk_size: int) -> Iterator[list[str]]:
        r"""Chunk a list into smaller lists of a given size.

        Args:
            lst: The list to chunk.
            chunk_size: The size of each chunk.

        Returns:
            An iterator over the chunks.

        Examples:
            >>> all_chunks = list(chunks([1, 2, 3, 4, 5], 2))
        """
        for i in range(0, len(lst), chunk_size):
            yield lst[i : i + chunk_size]

## `download_tokenizer_from_hf_hub(repo_id, cache_dir=None, token=None, revision=None, force_download=False, local_files_only=False)`

Download the tokenizer file of a Mistral model from the Hugging Face Hub.

See here for a list of our OSS models.

Note

You need to install the `huggingface_hub` package to use this method.

Please run `pip install mistral-common[hf-hub]` to install it.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`repo_id` | `str` | The Hugging Face repo ID. | required
`cache_dir` | `str | Path | None` | The directory where the tokenizer will be cached. | `None`
`token` | `bool | str | None` | The Hugging Face token to use to download the tokenizer. | `None`
`revision` | `str | None` | The revision of the model to use. If `None`, the latest revision will be used. | `None`
`force_download` | `bool` | Whether to force the download of the tokenizer. If `True`, the tokenizer will be downloaded even if it is already cached. | `False`
`local_files_only` | `bool` | Whether to only use local files. If `True`, the tokenizer will be downloaded only if it is already cached. | `False`

Returns:

Type | Description
--- | ---
`str` | The downloaded tokenizer local path for the given model ID.

Source code in `src/mistral_common/tokens/tokenizers/utils.py`

    132
    133
    134
    135
    136
    137
    138
    139
    140
    141
    142
    143
    144
    145
    146
    147
    148
    149
    150
    151
    152
    153
    154
    155
    156
    157
    158
    159
    160
    161
    162
    163
    164
    165
    166
    167
    168
    169
    170
    171
    172
    173
    174
    175
    176
    177
    178
    179
    180
    181
    182
    183
    184
    185
    186
    187
    188
    189
    190
    191
    192
    193
    194
    195
    196
    197
    198
    199
    200
    201
    202
    203
    204
    205
    206
    207
    208
    209
    210
    211
    212
    213

    def download_tokenizer_from_hf_hub(
        repo_id: str,
        cache_dir: str | Path | None = None,
        token: bool | str | None = None,
        revision: str | None = None,
        force_download: bool = False,
        local_files_only: bool = False,
    ) -> str:
        r"""Download the tokenizer file of a Mistral model from the Hugging Face Hub.

        See [here](https://huggingface.co/mistralai/models) for a list of our OSS models.

        Note:
            You need to install the `huggingface_hub` package to use this method.

            Please run `pip install mistral-common[hf-hub]` to install it.

        Args:
            repo_id: The Hugging Face repo ID.
            cache_dir: The directory where the tokenizer will be cached.
            token: The Hugging Face token to use to download the tokenizer.
            revision: The revision of the model to use. If `None`, the latest revision will be used.
            force_download: Whether to force the download of the tokenizer. If `True`, the tokenizer will be downloaded
                even if it is already cached.
            local_files_only: Whether to only use local files. If `True`, the tokenizer will be downloaded only if it is
                already cached.

        Returns:
            The downloaded tokenizer local path for the given model ID.
        """
        _assert_hub_installed()

        if force_download and local_files_only:
            raise ValueError("You cannot force the download of the tokenizer if you only want to use local files.")

        if not local_files_only:
            try:
                hf_api = huggingface_hub.HfApi()
                repo_files = hf_api.list_repo_files(repo_id, revision=revision, token=token)
                local_files_only = False
            except (
                requests.ConnectionError,
                requests.HTTPError,
                requests.Timeout,
                huggingface_hub.errors.OfflineModeIsEnabled,
            ) as e:
                if force_download:
                    raise e

                repo_files = list_local_hf_repo_files(repo_id=repo_id, revision=revision)
                local_files_only = True

                logger.info("Could not connect to the Hugging Face Hub. Using local files only.")

                if len(repo_files) == 0:
                    raise FileNotFoundError(
                        f"Could not connect to the Hugging Face Hub and no local files were found for the repo ID {repo_id}"
                        f" and revision {revision}. Please check your internet connection and try again."
                    ) from e
        else:
            repo_files = list_local_hf_repo_files(repo_id=repo_id, revision=revision)
            if len(repo_files) == 0:
                raise FileNotFoundError(
                    f"No local files found for the repo ID {repo_id} and revision {revision}. Please check the repo ID and"
                    " the revision or try to download the tokenizer without setting `local_files_only` to `True`."
                )

        try:
            tokenizer_file = get_one_valid_tokenizer_file(files=repo_files)
        except ValueError:
            raise ValueError(f"No valid tokenizer file found in the repo {repo_id}.")

        tokenizer_path = huggingface_hub.hf_hub_download(
            repo_id=repo_id,
            cache_dir=cache_dir,
            filename=tokenizer_file,
            token=token,
            revision=revision,
            local_files_only=local_files_only,
            force_download=force_download,
        )
        return tokenizer_path

## `get_one_valid_tokenizer_file(files)`

Get one valid tokenizer file from a list of files.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`files` | `list[str]` | The list of files to filter. | required

Returns:

Type | Description
--- | ---
`str` | The path to the tokenizer file.

Source code in `src/mistral_common/tokens/tokenizers/utils.py`

    106
    107
    108
    109
    110
    111
    112
    113
    114
    115
    116
    117
    118
    119
    120
    121
    122
    123
    124
    125
    126
    127
    128
    129

    def get_one_valid_tokenizer_file(files: list[str]) -> str:
        r"""Get one valid tokenizer file from a list of files.

        Args:
            files: The list of files to filter.

        Returns:
            The path to the tokenizer file.
        """
        valid_tokenizer_file_names_and_files = _filter_valid_tokenizer_files(files)

        if len(valid_tokenizer_file_names_and_files) == 0:
            raise ValueError("No tokenizer file found.")
        # If there are multiple tokenizer files, we use tekken.json if it exists, otherwise the versioned one.
        if len(valid_tokenizer_file_names_and_files) > 1:
            for file_name, tokenizer_file in valid_tokenizer_file_names_and_files:
                if "tekken.json" == file_name:
                    return tokenizer_file
            tokenizer_file = sorted(valid_tokenizer_file_names_and_files, key=lambda x: x[0])[-1][1]
            logger.warning(f"Multiple valid tokenizer files found. Using {tokenizer_file}.")
        else:
            tokenizer_file = valid_tokenizer_file_names_and_files[0][1]

        return tokenizer_file

## `list_local_hf_repo_files(repo_id, revision)`

list the files of a local Hugging Face repo.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`repo_id` | `str` | The Hugging Face repo ID. | required
`revision` | `str | None` | The revision of the model to use. If `None`, the latest revision will be used. | required

Source code in `src/mistral_common/tokens/tokenizers/utils.py`

    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76

    def list_local_hf_repo_files(repo_id: str, revision: str | None) -> list[str]:
        r"""list the files of a local Hugging Face repo.

        Args:
            repo_id: The Hugging Face repo ID.
            revision: The revision of the model to use. If `None`, the latest revision will be used.
        """
        _assert_hub_installed()

        repo_cache = Path(huggingface_hub.constants.HF_HUB_CACHE) / huggingface_hub.constants.REPO_ID_SEPARATOR.join(
            ["models", *repo_id.split("/")]
        )

        if revision is None:
            revision = huggingface_hub.constants.DEFAULT_REVISION
        if not revision:
            return []

        # Snapshots are keyed by commit hash, so resolve a branch/tag revision (e.g. "main") via `refs`.
        revision_file = repo_cache / "refs" / revision
        if revision_file.is_file():
            revision = revision_file.read_text().strip()

        revision_dir = repo_cache / "snapshots" / revision
        if revision_dir.is_dir():
            return os.listdir(revision_dir)

        return []