Create and return a Config object based on the current settings.
(self)
| 1435 | return RoutingReplayConfig(routing_replay_args) |
| 1436 | |
| 1437 | def create_engine_config(self) -> FDConfig: |
| 1438 | """ |
| 1439 | Create and return a Config object based on the current settings. |
| 1440 | """ |
| 1441 | all_dict = asdict(self) |
| 1442 | model_cfg = ModelConfig(all_dict) |
| 1443 | |
| 1444 | if not model_cfg.is_unified_ckpt and hasattr(model_cfg, "tensor_parallel_size"): |
| 1445 | self.tensor_parallel_size = model_cfg.tensor_parallel_size |
| 1446 | |
| 1447 | speculative_cfg = self.create_speculative_config() |
| 1448 | if not self.enable_chunked_prefill: |
| 1449 | if (current_platform.is_cuda() or current_platform.is_maca()) and self.splitwise_role == "mixed": |
| 1450 | # default enable chunked prefill |
| 1451 | self.enable_chunked_prefill = True |
| 1452 | |
| 1453 | self.disable_chunked_prefill = int(envs.FD_DISABLE_CHUNKED_PREFILL) |
| 1454 | if self.disable_chunked_prefill: |
| 1455 | self.enable_chunked_prefill = False |
| 1456 | |
| 1457 | if self.max_num_batched_tokens is None: |
| 1458 | if int(envs.ENABLE_V1_KVCACHE_SCHEDULER): |
| 1459 | if current_platform.is_maca(): |
| 1460 | self.max_num_batched_tokens = self.max_model_len |
| 1461 | else: |
| 1462 | self.max_num_batched_tokens = 8192 # if set to max_model_len, it's easy to be OOM |
| 1463 | else: |
| 1464 | if self.enable_chunked_prefill: |
| 1465 | self.max_num_batched_tokens = 2048 |
| 1466 | else: |
| 1467 | self.max_num_batched_tokens = self.max_model_len |
| 1468 | |
| 1469 | all_dict = asdict(self) |
| 1470 | all_dict["model_cfg"] = model_cfg |
| 1471 | cache_cfg = CacheConfig(all_dict) |
| 1472 | load_cfg = LoadConfig(all_dict) |
| 1473 | parallel_cfg = ParallelConfig(all_dict) |
| 1474 | scheduler_cfg = self.create_scheduler_config() |
| 1475 | graph_opt_cfg = self.create_graph_optimization_config() |
| 1476 | plas_attention_config = self.create_plas_attention_config() |
| 1477 | eplb_cfg = self.create_eplb_config() |
| 1478 | routing_replay_config = self.create_routing_repaly_config() |
| 1479 | router_config = RouterConfig(all_dict) |
| 1480 | |
| 1481 | early_stop_cfg = self.create_early_stop_config() |
| 1482 | early_stop_cfg.update_enable_early_stop(self.enable_early_stop) |
| 1483 | structured_outputs_config: StructuredOutputsConfig = StructuredOutputsConfig(args=all_dict) |
| 1484 | |
| 1485 | return FDConfig( |
| 1486 | model_config=model_cfg, |
| 1487 | scheduler_config=scheduler_cfg, |
| 1488 | tokenizer=self.tokenizer, |
| 1489 | cache_config=cache_cfg, |
| 1490 | load_config=load_cfg, |
| 1491 | parallel_config=parallel_cfg, |
| 1492 | speculative_config=speculative_cfg, |
| 1493 | eplb_config=eplb_cfg, |
| 1494 | structured_outputs_config=structured_outputs_config, |