Skip to content

# `mistral_common.tokens.tokenizers.instruct`

## `InstructTokenizerBase(tokenizer, image_encoder=None, audio_encoder=None)`

Bases: `InstructTokenizer`, `Generic[InstructRequestType, FIMRequestType, TokenizedType, AssistantMessageType]`

Base instruct tokenizer.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`tokenizer` | `Tokenizer` | The tokenizer to use. | required
`image_encoder` | `ImageEncoder | None` | The image encoder to use if any. | `None`
`audio_encoder` | `AudioEncoder | None` | The audio encoder to use. | `None`

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73

    def __init__(
        self,
        tokenizer: Tokenizer,
        image_encoder: ImageEncoder | None = None,
        audio_encoder: AudioEncoder | None = None,
    ):
        r"""Initialize the instruct tokenizer.

        Args:
            tokenizer: The tokenizer to use.
            image_encoder: The image encoder to use if any.
            audio_encoder: The audio encoder to use.
        """
        self.tokenizer = tokenizer
        self.image_encoder = image_encoder
        self.audio_encoder = audio_encoder
        super().__init__(tokenizer, image_encoder, audio_encoder)

### `decode(tokens, special_token_policy=SpecialTokenPolicy.IGNORE)`

Decode tokens to a string.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`tokens` | `list[int]` | The tokens to decode. | required
`special_token_policy` | `SpecialTokenPolicy` | The policy to use for special tokens. | `IGNORE`

Returns:

Type | Description
--- | ---
`str` | The decoded string.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    236
    237
    238
    239
    240
    241
    242
    243
    244
    245
    246

    def decode(self, tokens: list[int], special_token_policy: SpecialTokenPolicy = SpecialTokenPolicy.IGNORE) -> str:
        r"""Decode tokens to a string.

        Args:
            tokens: The tokens to decode.
            special_token_policy: The policy to use for special tokens.

        Returns:
            The decoded string.
        """
        return self.tokenizer.decode(tokens, special_token_policy=special_token_policy)

### `encode_assistant_message(message, is_before_last_user_message, continue_message)` `abstractmethod`

Encode an assistant message.

Raises:

Type | Description
--- | ---
`NotImplementedError` | The assistant message is not implemented for the base tokenizer.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    116
    117
    118
    119
    120
    121
    122
    123
    124
    125

    @abstractmethod
    def encode_assistant_message(
        self, message: AssistantMessageType, is_before_last_user_message: bool, continue_message: bool
    ) -> list[int]:
        r"""Encode an assistant message.

        Raises:
            NotImplementedError: The assistant message is not implemented for the base tokenizer.
        """
        raise NotImplementedError("Assistant message not implemented")

### `encode_instruct(request)`

Encode an instruct request.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`request` | `InstructRequest[AssistantMessageType, Tool]` | The request to encode. | required

Returns:

Type | Description
--- | ---
`Tokenized` | The encoded tokens.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    151
    152
    153
    154
    155
    156
    157
    158
    159
    160
    161
    162
    163
    164
    165
    166
    167
    168
    169
    170
    171
    172
    173
    174
    175
    176
    177
    178
    179
    180
    181
    182
    183
    184
    185
    186
    187
    188
    189
    190
    191
    192
    193
    194
    195
    196
    197
    198
    199
    200
    201
    202
    203
    204
    205
    206
    207
    208
    209
    210
    211
    212
    213
    214
    215
    216
    217
    218
    219
    220
    221
    222
    223
    224
    225
    226
    227
    228
    229
    230
    231
    232
    233
    234

    def encode_instruct(
        self,
        request: InstructRequest[AssistantMessageType, Tool],
    ) -> Tokenized:
        r"""Encode an instruct request.

        Args:
            request: The request to encode.

        Returns:
            The encoded tokens.
        """
        # init at bos
        images: list[np.ndarray] = []
        audios: list[Audio] = []
        prefix_ids: list[int] | None = None
        tokens_list: list[list[int] | None] = []

        # validate messages
        self.validate_messages(request.messages)

        # find last user message
        first_user_idx, last_user_idx = self.find_first_last_user(request)
        for msg_idx, msg in enumerate(request.messages):
            if (
                request.continue_final_message
                and (msg_idx == len(request.messages) - 1)
                and not isinstance(msg, AssistantMessage)
            ):
                raise InvalidMessageStructureException(
                    "Cannot continue final message if it is not an assistant message"
                )
            if isinstance(msg, UserMessage):
                new_tokens, new_images, new_audios = self.encode_user_message(
                    msg,
                    request.available_tools,
                    msg_idx == last_user_idx,
                    msg_idx == first_user_idx,
                    system_prompt=request.system_prompt,
                    force_img_first=True,  # img is always first when providing text/img chunk pair
                    settings=request.settings,
                )
                images.extend(new_images)
                audios.extend(new_audios)
            elif isinstance(msg, ToolMessage):
                new_tokens, new_images, new_audios = self.encode_tool_message(msg, msg_idx < last_user_idx)
                images.extend(new_images)
                audios.extend(new_audios)
            elif isinstance(msg, AssistantMessage):
                continue_message = request.continue_final_message and (msg_idx == len(request.messages) - 1)

                new_tokens = self.encode_assistant_message(
                    msg, msg_idx < last_user_idx, continue_message=continue_message
                )
                if msg_idx == len(request.messages) - 1:
                    prefix_ids = new_tokens
            elif isinstance(msg, SystemMessage):
                new_tokens, new_audios = self.encode_system_message(msg)
                audios.extend(new_audios)
            else:
                raise TokenizerException(f"Unknown message type {type(msg)}")

            tokens_list.append(new_tokens)

        if request.truncate_at_max_tokens is not None:
            self._truncate_for_max_tokens(
                tokens_list,
                request.messages,
                request.truncate_at_max_tokens,
                last_user_idx,
            )
        tokens = self.start()

        for tok in tokens_list:
            if tok is not None:
                tokens.extend(tok)

        return Tokenized(
            tokens=tokens,
            text=self.decode(tokens, special_token_policy=SpecialTokenPolicy.KEEP),
            prefix_ids=prefix_ids,
            images=images,
            audios=audios,
        )

### `encode_think(chunk)` `abstractmethod`

Encode a think chunk.

Raises:

Type | Description
--- | ---
`NotImplementedError` | The think chunk is not implemented for the base tokenizer.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    127
    128
    129
    130
    131
    132
    133
    134

    @abstractmethod
    def encode_think(self, chunk: ThinkChunk) -> list[int]:
        r"""Encode a think chunk.

        Raises:
            NotImplementedError: The think chunk is not implemented for the base tokenizer.
        """
        raise NotImplementedError("Think chunk not implemented")

### `encode_tool_message(message, is_before_last_user_message)` `abstractmethod`

Encode a tool message.

Raises:

Type | Description
--- | ---
`NotImplementedError` | The tool message is not implemented for the base tokenizer.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    105
    106
    107
    108
    109
    110
    111
    112
    113
    114

    @abstractmethod
    def encode_tool_message(
        self, message: ToolMessage, is_before_last_user_message: bool
    ) -> tuple[list[int], list[np.ndarray], list[Audio]]:
        r"""Encode a tool message.

        Raises:
            NotImplementedError: The tool message is not implemented for the base tokenizer.
        """
        raise NotImplementedError("Tool message not implemented")

### `find_first_last_user(request)` `staticmethod`

Find the first and last user message in the request.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`request` | `InstructRequest` | The request to search for user messages. | required

Returns:

Type | Description
--- | ---
`tuple[int, int]` | The index of the first and last user message.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

     86
     87
     88
     89
     90
     91
     92
     93
     94
     95
     96
     97
     98
     99
    100
    101
    102
    103

    @staticmethod
    def find_first_last_user(request: InstructRequest) -> tuple[int, int]:
        r"""Find the first and last user message in the request.

        Args:
            request: The request to search for user messages.

        Returns:
            The index of the first and last user message.
        """
        last_user_idx = -1
        first_user_idx = -1
        for i, msg in list(enumerate(request.messages)):
            if isinstance(msg, UserMessage):
                if first_user_idx == -1:
                    first_user_idx = i
                last_user_idx = i
        return first_user_idx, last_user_idx

### `start()`

Return the start tokens.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    82
    83
    84

    def start(self) -> list[int]:
        r"""Return the start tokens."""
        return [self.tokenizer.bos_id]

## `InstructTokenizerV1(tokenizer, image_encoder=None, audio_encoder=None)`

Bases: `InstructTokenizerBase`, `Generic[InstructRequestType, FIMRequestType, TokenizedType, AssistantMessageType]`

Instruct tokenizer V1.

This tokenizer has basic for messages. It does not support tools or image inputs.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73

    def __init__(
        self,
        tokenizer: Tokenizer,
        image_encoder: ImageEncoder | None = None,
        audio_encoder: AudioEncoder | None = None,
    ):
        r"""Initialize the instruct tokenizer.

        Args:
            tokenizer: The tokenizer to use.
            image_encoder: The image encoder to use if any.
            audio_encoder: The audio encoder to use.
        """
        self.tokenizer = tokenizer
        self.image_encoder = image_encoder
        self.audio_encoder = audio_encoder
        super().__init__(tokenizer, image_encoder, audio_encoder)

### `encode_assistant_message(message, is_before_last_user_message, continue_message)`

Encode an assistant message.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`message` | `AssistantMessageType` | The message to encode. | required
`is_before_last_user_message` | `bool` | Not used. | required
`continue_message` | `bool` | Whether to continue the message generation. Only use this if the assistant message is the last message. | required

Returns:

Type | Description
--- | ---
`list[int]` | The encoded tokens.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    335
    336
    337
    338
    339
    340
    341
    342
    343
    344
    345
    346
    347
    348
    349
    350
    351
    352
    353
    354
    355
    356
    357
    358
    359
    360
    361
    362
    363

    def encode_assistant_message(
        self, message: AssistantMessageType, is_before_last_user_message: bool, continue_message: bool
    ) -> list[int]:
        r"""Encode an assistant message.

        Args:
            message: The message to encode.
            is_before_last_user_message: Not used.
            continue_message: Whether to continue the message generation.
                Only use this if the assistant message is the last message.

        Returns:
            The encoded tokens.
        """
        assert isinstance(message, AssistantMessage), message
        if message.tool_calls is not None and len(message.tool_calls) > 0:
            raise TokenizerException("Tools not implemented for tokenizer V1")
        if continue_message and message.prefix:
            raise InvalidAssistantMessageException(
                "`continue_message` is only supported for assistant messages that have `prefix=False`."
            )
        elif message.content:
            assert isinstance(message.content, str), "Message content must be a string for tokenizer < V13"
            curr_tokens = self.tokenizer.encode(message.content, bos=False, eos=False)
        else:
            raise TokenizerException(f"{message.content} // {message.tool_calls}")
        if not message.prefix and not continue_message:
            curr_tokens.append(self.tokenizer.eos_id)
        return curr_tokens

### `encode_fim(request)`

Encode a FIM request.

Raises:

Type | Description
--- | ---
`TokenizerException` | The FIM request is not implemented for this version.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    373
    374
    375
    376
    377
    378
    379

    def encode_fim(self, request: FIMRequest) -> Tokenized:
        r"""Encode a FIM request.

        Raises:
           TokenizerException: The FIM request is not implemented for this version.
        """
        raise TokenizerException(f"FIM not available for {self.tokenizer.version}")

### `encode_think(chunk)`

Encode a think chunk.

Raises:

Type | Description
--- | ---
`TokenizerException` | The think chunk is not implemented for this version.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    365
    366
    367
    368
    369
    370
    371

    def encode_think(self, chunk: ThinkChunk) -> list[int]:
        r"""Encode a think chunk.

        Raises:
            TokenizerException: The think chunk is not implemented for this version.
        """
        raise TokenizerException("Think not implemented for tokenizer < V13.")

### `encode_tool_message(message, is_before_last_user_message)`

Encode a tool message.

Raises:

Type | Description
--- | ---
`TokenizerException` | The tool message is not implemented for this version.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    325
    326
    327
    328
    329
    330
    331
    332
    333

    def encode_tool_message(
        self, message: ToolMessage, is_before_last_user_message: bool
    ) -> tuple[list[int], list[np.ndarray], list[Audio]]:
        r"""Encode a tool message.

        Raises:
            TokenizerException: The tool message is not implemented for this version.
        """
        raise TokenizerException("Tools not implemented for tokenizer V1")

### `encode_user_content(content, is_last, system_prompt=None, force_img_first=False)`

Encode a user content.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`content` | `str | list[ContentChunk]` | The content to encode. | required
`is_last` | `bool` | Whether the message is the last one. | required
`system_prompt` | `str | None` | The system prompt. | `None`
`force_img_first` | `bool` | Not used. | `False`

Returns:

Type | Description
--- | ---
`tuple[list[int], list[ndarray], list[Audio]]` | The encoded tokens and empty list.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    299
    300
    301
    302
    303
    304
    305
    306
    307
    308
    309
    310
    311
    312
    313
    314
    315
    316
    317
    318
    319
    320
    321
    322
    323

    def encode_user_content(
        self,
        content: str | list[ContentChunk],
        is_last: bool,
        system_prompt: str | None = None,
        force_img_first: bool = False,
    ) -> tuple[list[int], list[np.ndarray], list[Audio]]:
        r"""Encode a user content.

        Args:
            content: The content to encode.
            is_last: Whether the message is the last one.
            system_prompt: The system prompt.
            force_img_first: Not used.

        Returns:
            The encoded tokens and empty list.
        """
        assert isinstance(content, str)

        if is_last and system_prompt:
            content = system_prompt + "\n\n" + content

        tokens = self.tokenizer.encode(content, bos=False, eos=False)
        return tokens, [], []

### `encode_user_message(message, available_tools, is_last, is_first, system_prompt=None, force_img_first=False, settings=ModelSettings.none())`

Encode a user message.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`message` | `UserMessage` | The message to encode. | required
`available_tools` | `list[Tool] | None` | Not used. | required
`is_last` | `bool` | Not used. | required
`is_first` | `bool` | Whether the message is the first one. | required
`system_prompt` | `str | None` | The system prompt. | `None`
`force_img_first` | `bool` | Not used. | `False`

Returns:

Type | Description
--- | ---
`tuple[list[int], list[ndarray], list[Audio]]` | The encoded tokens and empty list.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    260
    261
    262
    263
    264
    265
    266
    267
    268
    269
    270
    271
    272
    273
    274
    275
    276
    277
    278
    279
    280
    281
    282
    283
    284
    285
    286
    287
    288
    289
    290
    291
    292
    293
    294

    def encode_user_message(
        self,
        message: UserMessage,
        available_tools: list[Tool] | None,
        is_last: bool,
        is_first: bool,
        system_prompt: str | None = None,
        force_img_first: bool = False,
        settings: ModelSettings = ModelSettings.none(),
    ) -> tuple[list[int], list[np.ndarray], list[Audio]]:
        r"""Encode a user message.

        Args:
            message: The message to encode.
            available_tools: Not used.
            is_last: Not used.
            is_first: Whether the message is the first one.
            system_prompt: The system prompt.
            force_img_first: Not used.

        Returns:
            The encoded tokens and empty list.
        """
        assert isinstance(message.content, str), "Message content must be normalized"
        assert self.image_encoder is None, "InstructTokenizerV1 cannot encode images"

        content = ""
        if is_first and system_prompt:
            content = system_prompt + "\n\n" + message.content
        else:
            content = message.content

        message_txt = f"[INST] {content} [/INST]"
        curr_tokens, image, audio = self.encode_user_content(content=message_txt, is_last=False, system_prompt=None)
        return curr_tokens, image, audio

## `InstructTokenizerV11(tokenizer, image_encoder=None, audio_encoder=None)`

Bases: `InstructTokenizerV7`

Instruct tokenizer V11.

The difference with V7 tokenizer is that it encodes tool calls differently: Tool call results are encoded as : - [begin tool call] call_name_tokens [call id] call_id_tokens [args] content tokens

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    1250
    1251
    1252
    1253
    1254
    1255
    1256
    1257
    1258

    def __init__(
        self,
        tokenizer: Tokenizer,
        image_encoder: ImageEncoder | None = None,
        audio_encoder: AudioEncoder | None = None,
    ) -> None:
        super().__init__(tokenizer, image_encoder, audio_encoder)
        self.ARGS = self.tokenizer.get_special_token(SpecialTokens.args.value)
        self.CALL_ID = self.tokenizer.get_special_token(SpecialTokens.call_id.value)

## `InstructTokenizerV13(tokenizer, image_encoder=None, audio_encoder=None)`

Bases: `InstructTokenizerV11`

Instruct tokenizer V13.

The difference with V11 tokenizer is that it encodes tool calls differently

- available tools are tokenized at the first user message.
- call id is no longer tokenized for tool calls or results.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    1290
    1291
    1292
    1293
    1294
    1295
    1296
    1297
    1298
    1299
    1300
    1301
    1302
    1303
    1304
    1305
    1306

    def __init__(
        self,
        tokenizer: Tokenizer,
        image_encoder: ImageEncoder | None = None,
        audio_encoder: AudioEncoder | None = None,
    ) -> None:
        super().__init__(tokenizer, image_encoder, audio_encoder)
        assert isinstance(tokenizer, Tekkenizer), f"Tokenizer must be a Tekkenizer. Got {type(tokenizer)}"
        if (
            SpecialTokens.begin_think.value in tokenizer._special_tokens_reverse_vocab
            and SpecialTokens.end_think.value in tokenizer._special_tokens_reverse_vocab
        ):
            self.BEGIN_THINK: int | None = tokenizer.get_special_token(SpecialTokens.begin_think.value)
            self.END_THINK: int | None = tokenizer.get_special_token(SpecialTokens.end_think.value)
        else:
            self.BEGIN_THINK = None
            self.END_THINK = None

### `encode_think(chunk)`

Encode a thinking chunk.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`chunk` | `ThinkChunk` | The thinking chunk to encode. | required

Returns: The encoded tokens.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    1350
    1351
    1352
    1353
    1354
    1355
    1356
    1357
    1358
    1359
    1360
    1361
    1362
    1363
    1364

    def encode_think(self, chunk: ThinkChunk) -> list[int]:
        r"""Encode a thinking chunk.

        Args:
            chunk: The thinking chunk to encode.
        Returns:
            The encoded tokens.
        """
        assert self.BEGIN_THINK is not None, "think tokens are not available for this tokenizer."
        assert self.END_THINK is not None, "think tokens are not available for this tokenizer."
        tokens = self.tokenizer.encode(chunk.thinking, bos=False, eos=False)
        think_tokens = [self.BEGIN_THINK, *tokens]
        if chunk.closed:
            think_tokens.append(self.END_THINK)
        return think_tokens

### `encode_tool_message(message, is_before_last_user_message)`

Encode a tool message.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`message` | `ToolMessage` | The message to encode. | required
`is_before_last_user_message` | `bool` | Not used. | required

Returns: The encoded tokens, images, and audios.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    1323
    1324
    1325
    1326
    1327
    1328
    1329
    1330
    1331
    1332
    1333
    1334
    1335
    1336
    1337
    1338
    1339
    1340
    1341
    1342
    1343
    1344
    1345
    1346
    1347
    1348

    def encode_tool_message(
        self, message: ToolMessage, is_before_last_user_message: bool
    ) -> tuple[list[int], list[np.ndarray], list[Audio]]:
        r"""Encode a tool message.

        Args:
            message: The message to encode.
            is_before_last_user_message: Not used.
        Returns:
            The encoded tokens, images, and audios.
        """
        assert message.tool_call_id is not None, "Tool call id must be provided for tokenizer >= v13"

        if isinstance(message.content, str):
            content_tokens = self.tokenizer.encode(message.content, bos=False, eos=False)
            images: list[np.ndarray] = []
            audios: list[Audio] = []
        else:
            content_tokens, images, audios = self._encode_content_chunks(message.content)

        curr_tokens = [
            self.BEGIN_TOOL_RESULTS,
            *content_tokens,
            self.END_TOOL_RESULTS,
        ]
        return curr_tokens, images, audios

### `validate_messages(messages)` `classmethod`

Allows system prompts and audio chunks to coexist in v13.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    1366
    1367
    1368
    1369

    @classmethod
    def validate_messages(cls, messages: list[UATS]) -> None:
        r"""Allows system prompts and audio chunks to coexist in v13."""
        return

## `InstructTokenizerV15(tokenizer, image_encoder=None, audio_encoder=None)`

Bases: `InstructTokenizerV13`

Instruct tokenizer V15.

Extends V13 with model settings encoding. Inherits V13's tool call behavior (no call ID in tool calls/results, first-position tools, think support).

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    1379
    1380
    1381
    1382
    1383
    1384
    1385
    1386
    1387

    def __init__(
        self,
        tokenizer: Tokenizer,
        image_encoder: ImageEncoder | None = None,
        audio_encoder: AudioEncoder | None = None,
    ) -> None:
        super().__init__(tokenizer, image_encoder=image_encoder, audio_encoder=audio_encoder)
        self.BEGIN_MODEL_SETTINGS = self.tokenizer.get_special_token(SpecialTokens.begin_model_settings.value)
        self.END_MODEL_SETTINGS = self.tokenizer.get_special_token(SpecialTokens.end_model_settings.value)

### `encode_system_message(message)`

Encode a system message, rejecting ThinkChunk content.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    1413
    1414
    1415
    1416
    1417
    1418

    def encode_system_message(self, message: SystemMessage) -> tuple[list[int], list[Audio]]:
        r"""Encode a system message, rejecting ThinkChunk content."""
        if isinstance(message.content, list):
            if any(isinstance(chunk, ThinkChunk) for chunk in message.content):
                raise TokenizerException("ThinkChunk in system message is not supported for this model")
        return super().encode_system_message(message)

## `InstructTokenizerV2(tokenizer, image_encoder=None, audio_encoder=None)`

Bases: `InstructTokenizerV1`, `Generic[InstructRequestType, FIMRequestType, TokenizedType, AssistantMessageType]`

Instruct tokenizer V2.

This tokenizer adds supports to images, tools and FIM requests.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`tokenizer` | `Tokenizer` | The tokenizer to use. | required
`image_encoder` | `ImageEncoder | None` | The image encoder to use. | `None`
`audio_encoder` | `AudioEncoder | None` | The audio encoder to use. | `None`

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    398
    399
    400
    401
    402
    403
    404
    405
    406
    407
    408
    409
    410
    411
    412
    413
    414
    415
    416
    417
    418
    419
    420
    421

    def __init__(
        self,
        tokenizer: Tokenizer,
        image_encoder: ImageEncoder | None = None,
        audio_encoder: AudioEncoder | None = None,
    ):
        r"""Initialize the tokenizer.

        Args:
            tokenizer: The tokenizer to use.
            image_encoder: The image encoder to use.
            audio_encoder: The audio encoder to use.
        """
        super().__init__(tokenizer, image_encoder, audio_encoder)
        self.BEGIN_INST = self.tokenizer.get_special_token(SpecialTokens.begin_inst.value)
        self.END_INST = self.tokenizer.get_special_token(SpecialTokens.end_inst.value)
        self.BEGIN_AVAILABLE_TOOLS = self.tokenizer.get_special_token(SpecialTokens.begin_tools.value)
        self.END_AVAILABLE_TOOLS = self.tokenizer.get_special_token(SpecialTokens.end_tools.value)
        self.BEGIN_TOOL_RESULTS = self.tokenizer.get_special_token(SpecialTokens.begin_tool_results.value)
        self.END_TOOL_RESULTS = self.tokenizer.get_special_token(SpecialTokens.end_tool_results.value)
        self.TOOL_CALLS = self.tokenizer.get_special_token(SpecialTokens.tool_calls.value)
        self.BOS = self.tokenizer.get_special_token(SpecialTokens.bos.value)
        self.PREFIX = self.tokenizer.get_special_token(SpecialTokens.prefix.value)
        self.SUFFIX = self.tokenizer.get_special_token(SpecialTokens.suffix.value)

### `encode_assistant_message(message, is_before_last_user_message, continue_message)`

Encode an assistant message.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`message` | `AssistantMessageType` | The message to encode. | required
`is_before_last_user_message` | `bool` | Whether the message is before the last user message. If has tools and true, the message is not encoded. | required
`continue_message` | `bool` | Whether to continue the message generation. Only use this if the assistant message is the last message. | required

Returns:

Type | Description
--- | ---
`list[int]` | The encoded tokens.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    565
    566
    567
    568
    569
    570
    571
    572
    573
    574
    575
    576
    577
    578
    579
    580
    581
    582
    583
    584
    585
    586
    587
    588
    589
    590
    591
    592
    593
    594
    595
    596
    597
    598
    599

    def encode_assistant_message(
        self, message: AssistantMessageType, is_before_last_user_message: bool, continue_message: bool
    ) -> list[int]:
        r"""Encode an assistant message.

        Args:
            message: The message to encode.
            is_before_last_user_message: Whether the message is before the last user message. If has tools and true, the
                message is not encoded.
            continue_message: Whether to continue the message generation.
                Only use this if the assistant message is the last message.

        Returns:
            The encoded tokens.
        """
        if message.tool_calls and message.content:
            raise ValueError(f"Cannot have tool calls and content defined in the same assistant message {message}")
        if continue_message and message.prefix:
            raise InvalidAssistantMessageException(
                "`continue_message` is only supported for assistant messages that have `prefix=False`."
            )

        if message.tool_calls:
            if is_before_last_user_message:
                # don't tokenize tool call before last user message
                return []
            curr_tokens = self._encode_tool_calls_in_assistant_message(message)
        elif message.content:
            assert isinstance(message.content, str), "Message content must be a string for tokenizer < V7"
            curr_tokens = self._encode_normal_content_assistant_message(message)
        else:
            raise TokenizerException(f"Invalid assistant message: {message.content}")
        if not message.prefix and not continue_message:
            curr_tokens.append(self.tokenizer.eos_id)
        return curr_tokens

### `encode_fim(request)`

Encode a FIM request.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`request` | `FIMRequest` | The request to encode. | required

Returns:

Type | Description
--- | ---
`Tokenized` | The encoded tokens.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    606
    607
    608
    609
    610
    611
    612
    613
    614
    615
    616
    617
    618
    619
    620
    621
    622
    623
    624

    def encode_fim(self, request: FIMRequest) -> Tokenized:
        r"""Encode a FIM request.

        Args:
            request: The request to encode.

        Returns:
            The encoded tokens.
        """
        prefix_tokens = self.tokenizer.encode(request.prompt, bos=False, eos=False)
        suffix_tokens = self._encode_infilling(request.suffix) if request.suffix else []
        tokens = [
            self.BOS,
            self.SUFFIX,
            *suffix_tokens,
            self.PREFIX,
            *prefix_tokens,
        ]
        return Tokenized(tokens=tokens, text=self.decode(tokens, special_token_policy=SpecialTokenPolicy.KEEP))

### `encode_tool_message(message, is_before_last_user_message)`

Encode a tool message.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`message` | `ToolMessage` | The message to encode. | required
`is_before_last_user_message` | `bool` | Whether the message is before the last user message. If true, the message is not encoded. | required

Returns:

Type | Description
--- | ---
`tuple[list[int], list[ndarray], list[Audio]]` | The encoded tokens, images, and audios.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    507
    508
    509
    510
    511
    512
    513
    514
    515
    516
    517
    518
    519
    520
    521
    522
    523
    524
    525
    526
    527
    528
    529
    530
    531

    def encode_tool_message(
        self, message: ToolMessage, is_before_last_user_message: bool
    ) -> tuple[list[int], list[np.ndarray], list[Audio]]:
        r"""Encode a tool message.

        Args:
            message: The message to encode.
            is_before_last_user_message: Whether the message is before the last user message. If true, the message is
                not encoded.

        Returns:
            The encoded tokens, images, and audios.
        """
        if is_before_last_user_message:
            # don't tokenize last tool response before last user msg
            return [], [], []

        # Currently only supports single tool results
        tool_result_str = json.dumps([self._prepare_tool_result(message)], ensure_ascii=False)
        curr_tokens = [
            self.BEGIN_TOOL_RESULTS,
            *self.tokenizer.encode(tool_result_str, bos=False, eos=False),
            self.END_TOOL_RESULTS,
        ]
        return curr_tokens, [], []

### `encode_user_message(message, available_tools, is_last, is_first, system_prompt=None, force_img_first=False, settings=ModelSettings.none())`

Encode a user message.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`message` | `UserMessage` | The message to encode. | required
`available_tools` | `list[Tool] | None` | The list of available tools if any. | required
`is_last` | `bool` | Whether the message is the last one. | required
`is_first` | `bool` | Not used. | required
`system_prompt` | `str | None` | The system prompt. | `None`
`force_img_first` | `bool` | Whether to force the image to be first. | `False`

Returns:

Type | Description
--- | ---
`tuple[list[int], list[ndarray], list[Audio]]` | The encoded tokens and the list of images.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    423
    424
    425
    426
    427
    428
    429
    430
    431
    432
    433
    434
    435
    436
    437
    438
    439
    440
    441
    442
    443
    444
    445
    446
    447
    448
    449
    450
    451
    452
    453
    454
    455
    456
    457
    458
    459
    460
    461
    462
    463
    464
    465
    466
    467
    468
    469
    470
    471
    472
    473
    474
    475
    476
    477
    478
    479
    480
    481

    def encode_user_message(
        self,
        message: UserMessage,
        available_tools: list[Tool] | None,
        is_last: bool,
        is_first: bool,
        system_prompt: str | None = None,
        force_img_first: bool = False,
        settings: ModelSettings = ModelSettings.none(),
    ) -> tuple[list[int], list[np.ndarray], list[Audio]]:
        r"""Encode a user message.

        Args:
            message: The message to encode.
            available_tools: The list of available tools if any.
            is_last: Whether the message is the last one.
            is_first: Not used.
            system_prompt: The system prompt.
            force_img_first: Whether to force the image to be first.

        Returns:
            The encoded tokens and the list of images.
        """
        do_encode_tools_settings = False
        do_encode_tools_settings |= is_first and (
            self._message_position_to_encode_tools_settings == UserMessagePosition.first
        )
        do_encode_tools_settings |= is_last and (
            self._message_position_to_encode_tools_settings == UserMessagePosition.last
        )
        tools_settings_tokens: list[int] = []

        if do_encode_tools_settings and available_tools:
            tools = [tool.model_dump(exclude={"function": {"strict": True}}) for tool in available_tools]
            tools_json_tokens = self.tokenizer.encode(json.dumps(tools, ensure_ascii=False), bos=False, eos=False)
            tools_settings_tokens.extend(
                [
                    self.BEGIN_AVAILABLE_TOOLS,
                    *tools_json_tokens,
                    self.END_AVAILABLE_TOOLS,
                ]
            )

        if do_encode_tools_settings:
            tools_settings_tokens.extend(self._encode_settings(settings=settings))

        tokens, image, audio = self.encode_user_content(
            content=message.content,
            is_last=is_last,
            system_prompt=system_prompt,
            force_img_first=force_img_first,
        )

        prefix_tokens = [*tools_settings_tokens, self.BEGIN_INST]
        suffix_tokens = [self.END_INST]

        curr_tokens = prefix_tokens + tokens + suffix_tokens

        return curr_tokens, image, audio

## `InstructTokenizerV3(tokenizer, image_encoder=None, audio_encoder=None)`

Bases: `InstructTokenizerV2`, `Generic[InstructRequestType, FIMRequestType, TokenizedType, AssistantMessageType]`

Instruct tokenizer V3.

The only difference with V2 tokenizer is that it encodes the tool messages differently.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`tokenizer` | `Tokenizer` | The tokenizer to use. | required
`image_encoder` | `ImageEncoder | None` | The image encoder to use. | `None`
`audio_encoder` | `AudioEncoder | None` | The audio encoder to use. | `None`

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    635
    636
    637
    638
    639
    640
    641
    642
    643
    644
    645
    646
    647
    648

    def __init__(
        self,
        tokenizer: Tokenizer,
        image_encoder: ImageEncoder | None = None,
        audio_encoder: AudioEncoder | None = None,
    ):
        r"""Initialize the tokenizer.

        Args:
            tokenizer: The tokenizer to use.
            image_encoder: The image encoder to use.
            audio_encoder: The audio encoder to use.
        """
        super().__init__(tokenizer, image_encoder=image_encoder, audio_encoder=audio_encoder)

### `encode_assistant_message(message, is_before_last_user_message, continue_message)`

Encode an assistant message.

Note

Same as V2 but always encode the tool history. continue_message: Whether to continue the message generation. Only use this if the assistant message is the last message.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`message` | `AssistantMessageType` | The message to encode. | required
`is_before_last_user_message` | `bool` | Not used. | required

Returns:

Type | Description
--- | ---
`list[int]` | The encoded tokens.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    694
    695
    696
    697
    698
    699
    700
    701
    702
    703
    704
    705
    706
    707
    708
    709
    710
    711
    712

    def encode_assistant_message(
        self, message: AssistantMessageType, is_before_last_user_message: bool, continue_message: bool
    ) -> list[int]:
        r"""Encode an assistant message.

        Note:
            Same as [V2][mistral_common.tokens.tokenizers.instruct.InstructTokenizerV2.encode_assistant_message] but
            always encode the tool history.
            continue_message: Whether to continue the message generation.
                Only use this if the assistant message is the last message.

        Args:
            message: The message to encode.
            is_before_last_user_message: Not used.

        Returns:
            The encoded tokens.
        """
        return super().encode_assistant_message(message, False, continue_message)

### `encode_tool_message(message, is_before_last_user_message)`

Encode a tool message.

Note

Same as V2 but tools are not wrapped in a list and the history is also tokenized.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`message` | `ToolMessage` | The message to encode. | required
`is_before_last_user_message` | `bool` | Whether the message is before the last user message. If true, the message is not encoded. | required

Returns:

Type | Description
--- | ---
`tuple[list[int], list[ndarray], list[Audio]]` | The encoded tokens, images, and audios.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    669
    670
    671
    672
    673
    674
    675
    676
    677
    678
    679
    680
    681
    682
    683
    684
    685
    686
    687
    688
    689
    690
    691
    692

    def encode_tool_message(
        self, message: ToolMessage, is_before_last_user_message: bool
    ) -> tuple[list[int], list[np.ndarray], list[Audio]]:
        r"""Encode a tool message.

        Note:
            Same as [V2][mistral_common.tokens.tokenizers.instruct.InstructTokenizerV2.encode_tool_message] but tools
            are not wrapped in a list and the history is also tokenized.

        Args:
            message: The message to encode.
            is_before_last_user_message: Whether the message is before the last user message. If true, the message is
                not encoded.

        Returns:
            The encoded tokens, images, and audios.
        """
        tool_result_str = json.dumps(self._prepare_tool_result(message), ensure_ascii=False)
        curr_tokens = [
            self.BEGIN_TOOL_RESULTS,
            *self.tokenizer.encode(tool_result_str, bos=False, eos=False),
            self.END_TOOL_RESULTS,
        ]
        return curr_tokens, [], []

### `encode_user_content(content, is_last, system_prompt=None, force_img_first=False)`

Encode a user content.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`content` | `str | list[ContentChunk]` | The content to encode. | required
`is_last` | `bool` | Whether the message is the last one. | required
`system_prompt` | `str | None` | The system prompt. | `None`
`force_img_first` | `bool` | Whether to force the image to be first. | `False`

Returns:

Type | Description
--- | ---
`tuple[list[int], list[ndarray], list[Audio]]` | The encoded tokens and the images.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    770
    771
    772
    773
    774
    775
    776
    777
    778
    779
    780
    781
    782
    783
    784
    785
    786
    787
    788
    789
    790
    791
    792
    793
    794
    795
    796
    797
    798
    799
    800
    801
    802
    803
    804
    805
    806
    807
    808
    809
    810
    811
    812
    813
    814
    815
    816
    817
    818
    819
    820
    821
    822
    823
    824

    def encode_user_content(
        self,
        content: str | list[ContentChunk],
        is_last: bool,
        system_prompt: str | None = None,
        force_img_first: bool = False,
    ) -> tuple[list[int], list[np.ndarray], list[Audio]]:
        r"""Encode a user content.

        Args:
            content: The content to encode.
            is_last: Whether the message is the last one.
            system_prompt: The system prompt.
            force_img_first: Whether to force the image to be first.

        Returns:
            The encoded tokens and the images.
        """
        if isinstance(content, str):
            return super().encode_user_content(content, is_last, system_prompt)

        tokens: list[int] = []
        images: list[np.ndarray] = []
        audio: list[Audio] = []

        if force_img_first:
            content = self._maybe_put_image_first(content)

        first_chunk = True
        for chunk in content:
            content_str = ""
            if first_chunk and is_last and system_prompt:
                first_chunk = False
                content_str = system_prompt + "\n\n"
                tokens += self.tokenizer.encode(content_str, bos=False, eos=False)

            if isinstance(chunk, (AudioChunk, AudioURLChunk)):
                assert not content_str, (
                    f"It is not possible that `content` is non-empty when chunk is of type {type(chunk)}."
                )
                chunk_tokens, maybe_image, chunk_audio = self._encode_content_chunk(chunk)
                assert maybe_image is None, f"Unexpected image for audio chunk {type(chunk).__name__}."
                audio.append(chunk_audio)
            elif isinstance(chunk, (ImageChunk, ImageURLChunk)):
                chunk_tokens, chunk_image, maybe_audio = self._encode_content_chunk(chunk)
                assert maybe_audio is None, f"Unexpected audio for image chunk {type(chunk).__name__}."
                images.append(chunk_image)
            else:
                chunk_tokens, maybe_image, maybe_audio = self._encode_content_chunk(chunk)
                assert maybe_image is None and maybe_audio is None, (
                    f"Unexpected image/audio for chunk {type(chunk).__name__}."
                )
            tokens.extend(chunk_tokens)

        return tokens, images, audio

## `InstructTokenizerV7(tokenizer, image_encoder=None, audio_encoder=None)`

Bases: `InstructTokenizerV3`

Instruct tokenizer V7.

The difference with V3 tokenizer is that it encodes the system prompts differently: - in V7 the system prompts are treated as separate SystemMessages - they are no longer prepended to the last user message - they are printed between special tokens

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`tokenizer` | `Tokenizer` | The tokenizer to use. | required
`image_encoder` | `ImageEncoder | None` | The image encoder to use. | `None`
`audio_encoder` | `AudioEncoder | None` | The audio encoder to use. | `None`

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    837
    838
    839
    840
    841
    842
    843
    844
    845
    846
    847
    848
    849
    850
    851
    852
    853
    854
    855
    856
    857
    858

    def __init__(
        self,
        tokenizer: Tokenizer,
        image_encoder: ImageEncoder | None = None,
        audio_encoder: AudioEncoder | None = None,
    ) -> None:
        r"""Initialize the tokenizer.

        Args:
            tokenizer: The tokenizer to use.
            image_encoder: The image encoder to use.
            audio_encoder: The audio encoder to use.
        """

        super().__init__(tokenizer, image_encoder, audio_encoder)
        self.BEGIN_SYSTEM = self.tokenizer.get_special_token(SpecialTokens.begin_system.value)
        self.END_SYSTEM = self.tokenizer.get_special_token(SpecialTokens.end_system.value)
        self.BEGIN_TOOL_CONTENT = self.tokenizer.get_special_token(SpecialTokens.begin_tool_content.value)

        self.TRANSCRIBE = None
        if audio_encoder is not None and not audio_encoder.audio_config.is_streaming:
            self.TRANSCRIBE = self.tokenizer.get_special_token(SpecialTokens.transcribe.value)

### `encode_assistant_message(message, is_before_last_user_message, continue_message)`

Encode an assistant message.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`message` | `AssistantMessageType` | The message to encode. | required
`is_before_last_user_message` | `bool` | Not used. | required
`continue_message` | `bool` | Whether to continue the message generation. Only use this if the assistant message is the last message. | required

Returns:

Type | Description
--- | ---
`list[int]` | The encoded tokens.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    1148
    1149
    1150
    1151
    1152
    1153
    1154
    1155
    1156
    1157
    1158
    1159
    1160
    1161
    1162
    1163
    1164
    1165
    1166
    1167
    1168
    1169
    1170
    1171
    1172
    1173
    1174
    1175
    1176
    1177
    1178
    1179
    1180
    1181
    1182
    1183
    1184
    1185

    def encode_assistant_message(
        self, message: AssistantMessageType, is_before_last_user_message: bool, continue_message: bool
    ) -> list[int]:
        r"""Encode an assistant message.

        Args:
            message: The message to encode.
            is_before_last_user_message: Not used.
            continue_message: Whether to continue the message generation.
                Only use this if the assistant message is the last message.

        Returns:
            The encoded tokens.
        """
        if not message.content and not message.tool_calls:
            raise TokenizerException(f"Invalid assistant message: {message}")
        if continue_message and message.prefix:
            raise InvalidAssistantMessageException(
                "`continue_message` is only supported for assistant messages that have `prefix=False`."
            )

        curr_tokens: list = []
        if message.content:
            if isinstance(message.content, str):
                curr_tokens = self._encode_normal_content_assistant_message(message)
            elif isinstance(message.content, list):
                content_tokens, images, audios = self._encode_content_chunks(message.content)
                assert not images and not audios, (
                    f"Assistant messages cannot contain images or audios, got {len(images)} images "
                    f"and {len(audios)} audios."
                )
                curr_tokens += content_tokens
        if message.tool_calls:
            curr_tokens += self._encode_tool_calls_in_assistant_message(message)
        if not message.prefix and not continue_message:
            curr_tokens.append(self.tokenizer.eos_id)

        return curr_tokens

### `encode_speech_request(request)`

Encode a speech synthesis request into a tokenized sequence.

Produces: [BOS] + audio_tokens + [TEXT_TO_AUDIO] + text_tokens + [AUDIO_TO_TEXT] + [BEGIN_AUDIO].

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`request` | `SpeechRequest` | The speech request containing input text and voice/audio data. | required

Returns:

Type | Description
--- | ---
`Tokenized` | Tokenized object with the full token sequence and optional audio data.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    1213
    1214
    1215
    1216
    1217
    1218
    1219
    1220
    1221
    1222
    1223
    1224
    1225
    1226
    1227
    1228
    1229
    1230
    1231
    1232
    1233
    1234
    1235
    1236
    1237
    1238
    1239

    def encode_speech_request(self, request: SpeechRequest) -> Tokenized:
        r"""Encode a speech synthesis request into a tokenized sequence.

        Produces: [BOS] + audio_tokens + [TEXT_TO_AUDIO] + text_tokens + [AUDIO_TO_TEXT] + [BEGIN_AUDIO].

        Args:
            request: The speech request containing input text and voice/audio data.

        Returns:
            Tokenized object with the full token sequence and optional audio data.
        """
        assert self.audio_encoder is not None, (
            f"Audio encoder must be defined to encode audio, got {self.audio_encoder=}"
        )
        init_tokens = self.start()
        tokenized = Tokenized(tokens=init_tokens)
        tokenized_audio = self._encode_audio_for_speech_request(request.ref_audio, request.voice)
        tokenized.tokens.extend(tokenized_audio.tokens)
        tokenized.audios.extend(tokenized_audio.audios)
        tokens: list[int] = tokenized.tokens
        tokens.append(self.audio_encoder.text_to_audio_token)
        tokens.extend(self.tokenizer.encode(request.input, bos=False, eos=False))
        tokens.append(self.audio_encoder.audio_to_text_token)
        tokens.append(self.audio_encoder.begin_audio_token)
        tokenized.tokens = tokens

        return tokenized

### `encode_system_message(message)`

Encode a system message.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`message` | `SystemMessage` | The message to encode. | required

Returns:

Type | Description
--- | ---
`tuple[list[int], list[Audio]]` | The encoded tokens and audios.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    901
    902
    903
    904
    905
    906
    907
    908
    909
    910
    911
    912
    913
    914
    915
    916
    917

    def encode_system_message(self, message: SystemMessage) -> tuple[list[int], list[Audio]]:
        r"""Encode a system message.

        Args:
            message: The message to encode.

        Returns:
            The encoded tokens and audios.
        """
        tokens = [self.BEGIN_SYSTEM]
        if isinstance(content := message.content, str):
            content = [TextChunk(text=content)]
        content_tokens, images, audios = self._encode_content_chunks(content)
        assert not images, f"System messages cannot contain images, got {len(images)}."
        tokens += content_tokens
        tokens.append(self.END_SYSTEM)
        return tokens, audios

### `encode_tool_message(message, is_before_last_user_message)`

Encode a tool message.

Note

Same as V3 but tools are not wrapped in a list and history is also tokenized

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`message` | `ToolMessage` | The message to encode. | required
`is_before_last_user_message` | `bool` | Not used. | required

Returns:

Type | Description
--- | ---
`tuple[list[int], list[ndarray], list[Audio]]` | The encoded tokens, images, and audios.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    1115
    1116
    1117
    1118
    1119
    1120
    1121
    1122
    1123
    1124
    1125
    1126
    1127
    1128
    1129
    1130
    1131
    1132
    1133
    1134
    1135
    1136
    1137
    1138
    1139
    1140
    1141
    1142
    1143
    1144
    1145
    1146

    def encode_tool_message(
        self, message: ToolMessage, is_before_last_user_message: bool
    ) -> tuple[list[int], list[np.ndarray], list[Audio]]:
        r"""Encode a tool message.

        Note:
            Same as [V3][mistral_common.tokens.tokenizers.instruct.InstructTokenizerV3.encode_tool_message]
            but tools are not wrapped in a list and history is also tokenized

        Args:
            message: The message to encode.
            is_before_last_user_message: Not used.

        Returns:
            The encoded tokens, images, and audios.
        """
        assert message.tool_call_id is not None
        assert isinstance(message.content, str), "Message content must be normalized"
        tool_call_id_tokens = self.tokenizer.encode(message.tool_call_id, bos=False, eos=False)
        tokens = self.tokenizer.encode(message.content, bos=False, eos=False)

        prefix_tokens = [
            self.BEGIN_TOOL_RESULTS,
            *tool_call_id_tokens,
            self.BEGIN_TOOL_CONTENT,
        ]
        curr_tokens = [
            *prefix_tokens,
            *tokens,
            self.END_TOOL_RESULTS,
        ]
        return curr_tokens, [], []

### `encode_transcription(request)`

Encodes an audio transcription request into a tokenized format.

This method processes a transcription request containing audio data, encodes the user message, and returns the tokenized output.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`request` | `TranscriptionRequest` | The transcription request object containing the audio data to be encoded. | required

Returns:

Type | Description
--- | ---
`Tokenized` | The tokenized representation of the audio data, including processed audio and tokens

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

     985
     986
     987
     988
     989
     990
     991
     992
     993
     994
     995
     996
     997
     998
     999
    1000
    1001
    1002
    1003
    1004
    1005
    1006
    1007
    1008

    def encode_transcription(self, request: TranscriptionRequest) -> Tokenized:
        r"""
        Encodes an audio transcription request into a tokenized format.

        This method processes a transcription request containing audio data,
        encodes the user message, and returns the tokenized output.

        Args:
            request: The transcription request object containing
                the audio data to be encoded.

        Returns:
            Tokenized: The tokenized representation of the audio data, including processed audio and tokens
        """
        assert self.audio_encoder is not None, f"Audio encoder must be defined, got {self.audio_encoder=}"
        if self.audio_encoder.audio_config.transcription_format == TranscriptionFormat.INSTRUCT:
            return self._encode_instruct_transcription(request)
        elif self.audio_encoder.audio_config.transcription_format == TranscriptionFormat.STREAMING:
            return self._encode_streaming_transcription(request)

        raise InvalidRequestException(
            "Transcription format should be one of 'instruct', 'streaming', got "
            f"{self.audio_encoder.audio_config.transcription_format=}."
        )

### `encode_user_content(content, is_last, system_prompt=None, force_img_first=False)`

Encode a user content.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`content` | `str | list[ContentChunk]` | The content to encode. | required
`is_last` | `bool` | Whether the message is the last one. | required
`system_prompt` | `str | None` | The system prompt. | `None`
`force_img_first` | `bool` | Whether to force the image to be first. | `False`

Returns:

Type | Description
--- | ---
`tuple[list[int], list[ndarray], list[Audio]]` | The encoded tokens and the images.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    919
    920
    921
    922
    923
    924
    925
    926
    927
    928
    929
    930
    931
    932
    933
    934
    935
    936
    937
    938
    939
    940
    941
    942
    943
    944
    945
    946

    def encode_user_content(
        self,
        content: str | list[ContentChunk],
        is_last: bool,
        system_prompt: str | None = None,
        force_img_first: bool = False,
    ) -> tuple[list[int], list[np.ndarray], list[Audio]]:
        r"""Encode a user content.

        Args:
            content: The content to encode.
            is_last: Whether the message is the last one.
            system_prompt: The system prompt.
            force_img_first: Whether to force the image to be first.

        Returns:
            The encoded tokens and the images.
        """
        assert system_prompt is None, "in Tokenizer V7 we don't encode system prompts in user messages"

        if isinstance(content, str):
            return super().encode_user_content(content, is_last, system_prompt)

        if force_img_first:
            content = self._maybe_put_image_first(content)

        tokens, images, audio = self._encode_content_chunks(content)
        return tokens, images, audio

### `encode_user_message(message, available_tools, is_last, is_first, system_prompt=None, force_img_first=False, settings=ModelSettings.none())`

Encode a user message.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`message` | `UserMessage` | The message to encode. | required
`available_tools` | `list[Tool] | None` | The list of available tools if any. | required
`is_last` | `bool` | Whether the message is the last one. | required
`is_first` | `bool` | Whether the message is the first one. | required
`system_prompt` | `str | None` | Not used. | `None`
`force_img_first` | `bool` | Whether to force the image to be first. | `False`

Returns:

Type | Description
--- | ---
`tuple[list[int], list[ndarray], list[Audio]]` | The encoded tokens and the list of images.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    948
    949
    950
    951
    952
    953
    954
    955
    956
    957
    958
    959
    960
    961
    962
    963
    964
    965
    966
    967
    968
    969
    970
    971
    972
    973
    974
    975
    976
    977
    978
    979
    980
    981
    982
    983

    def encode_user_message(
        self,
        message: UserMessage,
        available_tools: list[Tool] | None,
        is_last: bool,
        is_first: bool,
        system_prompt: str | None = None,
        force_img_first: bool = False,
        settings: ModelSettings = ModelSettings.none(),
    ) -> tuple[list[int], list[np.ndarray], list[Audio]]:
        r"""Encode a user message.

        Args:
            message: The message to encode.
            available_tools: The list of available tools if any.
            is_last: Whether the message is the last one.
            is_first: Whether the message is the first one.
            system_prompt: Not used.
            force_img_first: Whether to force the image to be first.

        Returns:
            The encoded tokens and the list of images.
        """
        assert system_prompt is None, "in Tokenizer V7 we don't encode system prompts in user messages"

        tokens, images, audio = super().encode_user_message(
            message,
            available_tools,
            is_last=is_last,
            is_first=is_first,
            system_prompt=None,
            force_img_first=force_img_first,
            settings=settings,
        )

        return tokens, images, audio

### `validate_messages(messages)` `classmethod`

Validates that system prompts and audio chunks are not used together in v7.

Source code in `src/mistral_common/tokens/tokenizers/instruct.py`

    1099
    1100
    1101
    1102
    1103
    1104

    @classmethod
    def validate_messages(cls, messages: list[UATS]) -> None:
        r"""Validates that system prompts and audio chunks are not used together in v7."""
        if cls._has_audio(messages):
            if any(isinstance(message, SystemMessage) for message in messages):
                raise ValueError("System messages are not yet allowed when audio is present")