Skip to content

# `mistral_common.experimental.app.main`

## `cli()`

Mistral-common CLI.

Source code in `src/mistral_common/experimental/app/main.py`

    67
    68
    69
    70

    @click.group()
    def cli() -> None:
        r"""Mistral-common CLI."""
        pass

## `create_app(tokenizer, validation_mode=ValidationMode.test, engine_url='127.0.0.1', engine_backend=EngineBackend.llama_cpp, timeout=60)`

Create a Mistral-common FastAPI app with the given tokenizer and validation mode.

Parameters:

Name | Type | Description | Default
--- | --- | --- | ---
`tokenizer` | `str | Path | MistralTokenizer` | The tokenizer path or a MistralTokenizer instance. | required
`validation_mode` | `ValidationMode` | The validation mode to use. | `test`
`engine_url` | `str` | The URL of the engine API. | `'127.0.0.1'`
`timeout` | `int` | The timeout of the engine API. | `60`

Returns:

Type | Description
--- | ---
`FastAPI` | The Mistral-common FastAPI app.

Source code in `src/mistral_common/experimental/app/main.py`

    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64

    def create_app(
        tokenizer: str | Path | MistralTokenizer,
        validation_mode: ValidationMode = ValidationMode.test,
        engine_url: str = "127.0.0.1",
        engine_backend: EngineBackend = EngineBackend.llama_cpp,
        timeout: int = 60,
    ) -> FastAPI:
        r"""Create a Mistral-common FastAPI app with the given tokenizer and validation mode.

        Args:
            tokenizer: The tokenizer path or a MistralTokenizer instance.
            validation_mode: The validation mode to use.
            engine_url: The URL of the engine API.
            timeout: The timeout of the engine API.

        Returns:
            The Mistral-common FastAPI app.
        """
        if not isinstance(tokenizer, (MistralTokenizer, str, Path)):
            raise ValueError("Tokenizer must be a path or a MistralTokenizer instance.")

        app = FastAPI()
        app.include_router(tokenize_router)
        app.include_router(decode_router)
        app.include_router(main_router)

        @lru_cache
        def get_settings_override() -> Settings:
            settings = Settings(
                engine_url=engine_url,
                engine_backend=engine_backend,
                timeout=timeout,
            )
            if isinstance(tokenizer, MistralTokenizer):
                settings.tokenizer = tokenizer
            else:
                settings._load_tokenizer(tokenizer, validation_mode)
            return settings

        get_settings_override()
        app.dependency_overrides[get_settings] = get_settings_override

        return app

## `serve(tokenizer_path, validation_mode=ValidationMode.test, host='127.0.0.1', port=0, engine_url='http://127.0.0.1:8080', engine_backend=EngineBackend.llama_cpp.value, timeout=60)`

Serve the Mistral-common API with the given tokenizer path and validation mode.

Source code in `src/mistral_common/experimental/app/main.py`

     73
     74
     75
     76
     77
     78
     79
     80
     81
     82
     83
     84
     85
     86
     87
     88
     89
     90
     91
     92
     93
     94
     95
     96
     97
     98
     99
    100
    101
    102
    103
    104
    105
    106
    107
    108
    109
    110
    111
    112
    113
    114
    115
    116
    117
    118
    119
    120
    121
    122
    123
    124
    125
    126
    127
    128
    129
    130
    131
    132

    @cli.command(name="serve", context_settings={"auto_envvar_prefix": "UVICORN"})
    @click.argument("tokenizer_path", type=str)
    @click.argument(
        "validation_mode",
        type=click.Choice([mode.value for mode in ValidationMode], case_sensitive=False),
        default=ValidationMode.test.value,
    )
    @click.option(
        "--host",
        type=str,
        default="127.0.0.1",
        help="Mistral-common API host",
        show_default=True,
    )
    @click.option(
        "--port",
        type=int,
        default=0,
        help="Mistral-common API port",
        show_default=True,
    )
    @click.option(
        "--engine-url",
        type=str,
        default="http://127.0.0.1:8080",
        help="Engine URL",
        show_default=True,
    )
    @click.option(
        "--engine-backend",
        type=click.Choice([mode.value for mode in EngineBackend], case_sensitive=False),
        default=EngineBackend.llama_cpp.value,
        help="Engine API backend",
        show_default=True,
    )
    @click.option(
        "--timeout",
        type=int,
        default=60,
        help="Timeout",
        show_default=True,
    )
    def serve(
        tokenizer_path: str | Path,
        validation_mode: ValidationMode | str = ValidationMode.test,
        host: str = "127.0.0.1",
        port: int = 0,
        engine_url: str = "http://127.0.0.1:8080",
        engine_backend: str = EngineBackend.llama_cpp.value,
        timeout: int = 60,
    ) -> None:
        r"""Serve the Mistral-common API with the given tokenizer path and validation mode."""
        app = create_app(
            tokenizer=tokenizer_path,
            validation_mode=ValidationMode(validation_mode),
            engine_url=engine_url,
            engine_backend=EngineBackend(engine_backend),
            timeout=timeout,
        )
        uvicorn.run(app, host=host, port=port)