| 39 | |
| 40 | |
| 41 | class ServerProcess: |
| 42 | # default options |
| 43 | debug: bool = False |
| 44 | server_port: int = 8080 |
| 45 | server_host: str = "127.0.0.1" |
| 46 | model_hf_repo: str = "ggml-org/models" |
| 47 | model_hf_file: str | None = "tinyllamas/stories260K.gguf" |
| 48 | model_alias: str = "tinyllama-2" |
| 49 | temperature: float = 0.8 |
| 50 | seed: int = 42 |
| 51 | |
| 52 | # custom options |
| 53 | model_alias: str | None = None |
| 54 | model_url: str | None = None |
| 55 | model_file: str | None = None |
| 56 | model_draft: str | None = None |
| 57 | n_threads: int | None = None |
| 58 | n_gpu_layer: int | None = None |
| 59 | n_batch: int | None = None |
| 60 | n_ubatch: int | None = None |
| 61 | n_ctx: int | None = None |
| 62 | n_ga: int | None = None |
| 63 | n_ga_w: int | None = None |
| 64 | n_predict: int | None = None |
| 65 | n_prompts: int | None = 0 |
| 66 | slot_save_path: str | None = None |
| 67 | id_slot: int | None = None |
| 68 | cache_prompt: bool | None = None |
| 69 | n_slots: int | None = None |
| 70 | ctk: str | None = None |
| 71 | ctv: str | None = None |
| 72 | fa: bool | None = None |
| 73 | server_continuous_batching: bool | None = False |
| 74 | server_embeddings: bool | None = False |
| 75 | server_reranking: bool | None = False |
| 76 | server_metrics: bool | None = False |
| 77 | server_slots: bool | None = False |
| 78 | pooling: str | None = None |
| 79 | draft: int | None = None |
| 80 | api_key: str | None = None |
| 81 | lora_files: List[str] | None = None |
| 82 | disable_ctx_shift: int | None = False |
| 83 | draft_min: int | None = None |
| 84 | draft_max: int | None = None |
| 85 | no_webui: bool | None = None |
| 86 | jinja: bool | None = None |
| 87 | reasoning_format: Literal['deepseek', 'none', 'nothink'] | None = None |
| 88 | reasoning_budget: int | None = None |
| 89 | chat_template: str | None = None |
| 90 | chat_template_file: str | None = None |
| 91 | server_path: str | None = None |
| 92 | mmproj_url: str | None = None |
| 93 | |
| 94 | # session variables |
| 95 | process: subprocess.Popen | None = None |
| 96 | |
| 97 | def __init__(self): |
| 98 | if "N_GPU_LAYERS" in os.environ: |
no outgoing calls