The configuration class which contains all fastdeploy-related configuration. This simplifies passing around the distinct configurations in the codebase.
| 1696 | |
| 1697 | |
| 1698 | class FDConfig: |
| 1699 | """ |
| 1700 | The configuration class which contains all fastdeploy-related configuration. This |
| 1701 | simplifies passing around the distinct configurations in the codebase. |
| 1702 | """ |
| 1703 | |
| 1704 | def __init__( |
| 1705 | self, |
| 1706 | model_config: ModelConfig = None, |
| 1707 | cache_config: CacheConfig = None, |
| 1708 | parallel_config: ParallelConfig = None, |
| 1709 | load_config: LoadConfig = None, |
| 1710 | commit_config: CommitConfig = CommitConfig(), |
| 1711 | scheduler_config: SchedulerConfig = None, |
| 1712 | device_config: DeviceConfig = None, |
| 1713 | quant_config: QuantConfigBase = None, |
| 1714 | graph_opt_config: GraphOptimizationConfig = None, |
| 1715 | plas_attention_config: PlasAttentionConfig = None, |
| 1716 | speculative_config: SpeculativeConfig = None, |
| 1717 | eplb_config: EPLBConfig = None, |
| 1718 | structured_outputs_config: StructuredOutputsConfig = None, |
| 1719 | router_config: RouterConfig = None, |
| 1720 | tokenizer: str = None, |
| 1721 | ips: str = None, |
| 1722 | use_warmup: bool = False, |
| 1723 | limit_mm_per_prompt: Optional[Dict[str, Any]] = None, |
| 1724 | mm_processor_kwargs: Optional[Dict[str, Any]] = None, |
| 1725 | max_num_partial_prefills: int = 1, |
| 1726 | max_long_partial_prefills: int = 1, |
| 1727 | long_prefill_token_threshold: int = 0, |
| 1728 | early_stop_config: Optional[Dict[str, Any]] = None, |
| 1729 | tool_parser: str = None, |
| 1730 | test_mode=False, |
| 1731 | routing_replay_config: Optional[RoutingReplayConfig] = None, |
| 1732 | deploy_modality: "DeployModality" = None, |
| 1733 | ): |
| 1734 | self.model_config: ModelConfig = model_config # type: ignore |
| 1735 | self.cache_config: CacheConfig = cache_config # type: ignore |
| 1736 | self.scheduler_config: SchedulerConfig = scheduler_config # type: ignore |
| 1737 | self.parallel_config = parallel_config # type: ignore |
| 1738 | self.speculative_config: SpeculativeConfig = speculative_config |
| 1739 | self.eplb_config: Optional[EPLBConfig] = eplb_config |
| 1740 | self.device_config: DeviceConfig = device_config # type: ignore |
| 1741 | self.load_config: LoadConfig = load_config |
| 1742 | self.quant_config: Optional[QuantConfigBase] = quant_config |
| 1743 | self.graph_opt_config: Optional[GraphOptimizationConfig] = graph_opt_config |
| 1744 | self.early_stop_config: Optional[EarlyStopConfig] = early_stop_config |
| 1745 | self.plas_attention_config: Optional[PlasAttentionConfig] = plas_attention_config |
| 1746 | self.structured_outputs_config: StructuredOutputsConfig = structured_outputs_config |
| 1747 | self.router_config: RouterConfig = router_config |
| 1748 | self.routing_replay_config = routing_replay_config |
| 1749 | self.deploy_modality: DeployModality = deploy_modality if deploy_modality is not None else DeployModality.MIXED |
| 1750 | max_capture_shape = self.scheduler_config.max_num_seqs |
| 1751 | if self.speculative_config is not None and self.speculative_config.method in ["mtp", "suffix"]: |
| 1752 | max_capture_shape = self.scheduler_config.max_num_seqs * ( |
| 1753 | self.speculative_config.num_speculative_tokens + 1 |
| 1754 | ) |
| 1755 | assert max_capture_shape % 2 == 0, "CUDAGraph only supports capturing even token nums in MTP scenarios." |
no outgoing calls