| 43 | |
| 44 | |
| 45 | class ServerProcess: |
| 46 | # default options |
| 47 | debug: bool = False |
| 48 | server_port: int = 8080 |
| 49 | server_host: str = "127.0.0.1" |
| 50 | model_hf_repo: str | None = "ggml-org/models" |
| 51 | model_hf_file: str | None = "tinyllamas/stories260K.gguf" |
| 52 | model_alias: str = "tinyllama-2" |
| 53 | temperature: float = 0.8 |
| 54 | seed: int = 42 |
| 55 | offline: bool = False |
| 56 | |
| 57 | # custom options |
| 58 | model_alias: str | None = None |
| 59 | model_tags: str | None = None |
| 60 | model_url: str | None = None |
| 61 | model_file: str | None = None |
| 62 | model_draft: str | None = None |
| 63 | n_threads: int | None = None |
| 64 | n_gpu_layer: int | None = None |
| 65 | n_batch: int | None = None |
| 66 | n_ubatch: int | None = None |
| 67 | n_ctx: int | None = None |
| 68 | n_ga: int | None = None |
| 69 | n_ga_w: int | None = None |
| 70 | n_predict: int | None = None |
| 71 | n_prompts: int | None = 0 |
| 72 | slot_save_path: str | None = None |
| 73 | id_slot: int | None = None |
| 74 | cache_prompt: bool | None = None |
| 75 | n_slots: int | None = None |
| 76 | ctk: str | None = None |
| 77 | ctv: str | None = None |
| 78 | fa: str | None = None |
| 79 | server_continuous_batching: bool | None = False |
| 80 | server_embeddings: bool | None = False |
| 81 | server_reranking: bool | None = False |
| 82 | server_metrics: bool | None = False |
| 83 | kv_unified: bool | None = False |
| 84 | server_slots: bool | None = False |
| 85 | pooling: str | None = None |
| 86 | draft: int | None = None |
| 87 | api_key: str | None = None |
| 88 | models_dir: str | None = None |
| 89 | models_max: int | None = None |
| 90 | no_models_autoload: bool | None = None |
| 91 | lora_files: List[str] | None = None |
| 92 | enable_ctx_shift: int | None = False |
| 93 | draft_min: int | None = None |
| 94 | draft_max: int | None = None |
| 95 | no_webui: bool | None = None |
| 96 | jinja: bool | None = None |
| 97 | reasoning_format: Literal['deepseek', 'none', 'nothink'] | None = None |
| 98 | reasoning: Literal['on', 'off', 'auto'] | None = None |
| 99 | chat_template: str | None = None |
| 100 | chat_template_file: str | None = None |
| 101 | server_path: str | None = None |
| 102 | mmproj_url: str | None = None |
no outgoing calls