(
self,
)
| 204 | return engine_args |
| 205 | |
| 206 | def create_engine_configs( |
| 207 | self, |
| 208 | ) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]: |
| 209 | model_config = ModelConfig( |
| 210 | self.model, self.tokenizer, self.tokenizer_mode, |
| 211 | self.trust_remote_code, self.download_dir, self.load_format, |
| 212 | self.dtype, self.seed, self.attention_backend, self.revision, |
| 213 | self.tokenizer_revision, self.max_model_len, self.quantization) |
| 214 | cache_config = CacheConfig(self.block_size, |
| 215 | self.gpu_memory_utilization, |
| 216 | self.swap_space, |
| 217 | model_config.get_sliding_window()) |
| 218 | parallel_config = ParallelConfig(self.pipeline_parallel_size, |
| 219 | self.tensor_parallel_size, |
| 220 | self.worker_use_ray, |
| 221 | self.max_parallel_loading_workers) |
| 222 | scheduler_config = SchedulerConfig(self.max_num_batched_tokens, |
| 223 | self.max_num_seqs, |
| 224 | model_config.max_model_len, |
| 225 | self.pipeline_parallel_size, |
| 226 | self.profiler_path) |
| 227 | return model_config, cache_config, parallel_config, scheduler_config |
| 228 | |
| 229 | |
| 230 | @dataclass |
no test coverage detected