| 73 | |
| 74 | @dataclass |
| 75 | class EngineArgs: |
| 76 | # Model configuration parameters |
| 77 | model: str = "baidu/ernie-45-turbo" |
| 78 | """ |
| 79 | The name or path of the model to be used. |
| 80 | """ |
| 81 | port: Optional[str] = None |
| 82 | """ |
| 83 | Port for api server. |
| 84 | """ |
| 85 | metrics_port: Optional[str] = None |
| 86 | """ |
| 87 | Port for metrics server. |
| 88 | """ |
| 89 | served_model_name: Optional[str] = None |
| 90 | """ |
| 91 | The name of the model being served. |
| 92 | """ |
| 93 | revision: Optional[str] = "master" |
| 94 | """ |
| 95 | The revision for downloading models. |
| 96 | """ |
| 97 | model_config_name: Optional[str] = "config.json" |
| 98 | """ |
| 99 | The name of the model configuration file. |
| 100 | """ |
| 101 | tokenizer: str = None |
| 102 | """ |
| 103 | The name or path of the tokenizer (defaults to model path if not provided). |
| 104 | """ |
| 105 | tokenizer_base_url: str = None |
| 106 | """ |
| 107 | The base URL of the remote tokenizer service (used instead of local tokenizer if provided). |
| 108 | """ |
| 109 | max_model_len: int = 2048 |
| 110 | """ |
| 111 | Maximum context length supported by the model. |
| 112 | """ |
| 113 | tensor_parallel_size: int = 1 |
| 114 | """ |
| 115 | Degree of tensor parallelism. |
| 116 | """ |
| 117 | block_size: int = 64 |
| 118 | """ |
| 119 | Number of tokens in one processing block. |
| 120 | """ |
| 121 | task: TaskOption = "generate" |
| 122 | """ |
| 123 | The task to be executed by the model. |
| 124 | """ |
| 125 | runner: RunnerOption = "auto" |
| 126 | """ |
| 127 | The type of model runner to use.Each FD instance only supports one model runner. |
| 128 | even if the same model can be used for multiple types. |
| 129 | """ |
| 130 | convert: ConvertOption = "auto" |
| 131 | """ |
| 132 | Convert the model using adapters. The most common use case is to |
no outgoing calls