| 50 | |
| 51 | |
| 52 | class ServerConfig(pydra.Config): |
| 53 | model: str |
| 54 | tokenizer: str | None = None |
| 55 | |
| 56 | trust_remote_code: bool = False |
| 57 | dtype: str = "bfloat16" |
| 58 | rope_scaling: str | None = None |
| 59 | |
| 60 | use_hydragen: bool = False |
| 61 | hydragen_min_group_size: int = 32 |
| 62 | hydragen_min_prefix_len: int = 256 |
| 63 | |
| 64 | enable_chosen_logprobs: bool = True |
| 65 | max_topk_logprobs: int | None = None |
| 66 | |
| 67 | port: int = 10210 |
| 68 | local_proc_name: str = "server" |
| 69 | |
| 70 | log_level: str = "INFO" |
| 71 | log_procs: list[str] | None = None |
| 72 | uvicorn_log_level: str = "info" |
| 73 | |
| 74 | stats_report_seconds: float = 5.0 |
| 75 | statsd_server_url: None | str = None |
| 76 | |
| 77 | page_size: int = 16 |
| 78 | kv_cache_num_tokens: int = 1024 * 128 |
| 79 | |
| 80 | torch_compile: bool = False |
| 81 | |
| 82 | # the batch size at which we switch to using async TP |
| 83 | async_tp_threshold: int | None = None |
| 84 | |
| 85 | max_tokens_per_forward: int = 8192 |
| 86 | max_seqs_per_forward: int = 1024 |
| 87 | prefill_round_up_multiple: int = 16 |
| 88 | |
| 89 | scheduling_steps_ahead: int = 8 |
| 90 | stop_string_num_token_lookback: int = 5 |
| 91 | |
| 92 | dp_size: int = 1 |
| 93 | pp_size: int = 1 |
| 94 | tp_size: int = 1 |
| 95 | |
| 96 | # adding extra stages to hide the latency |
| 97 | # of sending lm-head results from the end of the pipeline to the start, |
| 98 | # as well as buffer data dependencies from sequences being rearranged |
| 99 | # across microbatches (e.g. as sequences finish / new sequences start). |
| 100 | pp_num_buffer_stages: int = 1 |
| 101 | |
| 102 | track_early_stopping: bool = True |
| 103 | early_stopping_buffer_size: int = 2048 |
| 104 | early_stopping_num_prediction_buckets: int = 1024 |
| 105 | early_stopping_initial_wait: int = 16 |
| 106 | early_stopping_init_mean: float | None = None |
| 107 | early_stopping_init_std: float | None = None |
| 108 | max_num_tokens_per_request: int | None = None |
| 109 |
no outgoing calls