(
self,
model_config: ModelConfig = None,
cache_config: CacheConfig = None,
parallel_config: ParallelConfig = None,
load_config: LoadConfig = None,
commit_config: CommitConfig = CommitConfig(),
scheduler_config: SchedulerConfig = None,
device_config: DeviceConfig = None,
quant_config: QuantConfigBase = None,
graph_opt_config: GraphOptimizationConfig = None,
plas_attention_config: PlasAttentionConfig = None,
speculative_config: SpeculativeConfig = None,
eplb_config: EPLBConfig = None,
structured_outputs_config: StructuredOutputsConfig = None,
router_config: RouterConfig = None,
tokenizer: str = None,
ips: str = None,
use_warmup: bool = False,
limit_mm_per_prompt: Optional[Dict[str, Any]] = None,
mm_processor_kwargs: Optional[Dict[str, Any]] = None,
max_num_partial_prefills: int = 1,
max_long_partial_prefills: int = 1,
long_prefill_token_threshold: int = 0,
early_stop_config: Optional[Dict[str, Any]] = None,
tool_parser: str = None,
test_mode=False,
routing_replay_config: Optional[RoutingReplayConfig] = None,
deploy_modality: "DeployModality" = None,
)
| 1702 | """ |
| 1703 | |
| 1704 | def __init__( |
| 1705 | self, |
| 1706 | model_config: ModelConfig = None, |
| 1707 | cache_config: CacheConfig = None, |
| 1708 | parallel_config: ParallelConfig = None, |
| 1709 | load_config: LoadConfig = None, |
| 1710 | commit_config: CommitConfig = CommitConfig(), |
| 1711 | scheduler_config: SchedulerConfig = None, |
| 1712 | device_config: DeviceConfig = None, |
| 1713 | quant_config: QuantConfigBase = None, |
| 1714 | graph_opt_config: GraphOptimizationConfig = None, |
| 1715 | plas_attention_config: PlasAttentionConfig = None, |
| 1716 | speculative_config: SpeculativeConfig = None, |
| 1717 | eplb_config: EPLBConfig = None, |
| 1718 | structured_outputs_config: StructuredOutputsConfig = None, |
| 1719 | router_config: RouterConfig = None, |
| 1720 | tokenizer: str = None, |
| 1721 | ips: str = None, |
| 1722 | use_warmup: bool = False, |
| 1723 | limit_mm_per_prompt: Optional[Dict[str, Any]] = None, |
| 1724 | mm_processor_kwargs: Optional[Dict[str, Any]] = None, |
| 1725 | max_num_partial_prefills: int = 1, |
| 1726 | max_long_partial_prefills: int = 1, |
| 1727 | long_prefill_token_threshold: int = 0, |
| 1728 | early_stop_config: Optional[Dict[str, Any]] = None, |
| 1729 | tool_parser: str = None, |
| 1730 | test_mode=False, |
| 1731 | routing_replay_config: Optional[RoutingReplayConfig] = None, |
| 1732 | deploy_modality: "DeployModality" = None, |
| 1733 | ): |
| 1734 | self.model_config: ModelConfig = model_config # type: ignore |
| 1735 | self.cache_config: CacheConfig = cache_config # type: ignore |
| 1736 | self.scheduler_config: SchedulerConfig = scheduler_config # type: ignore |
| 1737 | self.parallel_config = parallel_config # type: ignore |
| 1738 | self.speculative_config: SpeculativeConfig = speculative_config |
| 1739 | self.eplb_config: Optional[EPLBConfig] = eplb_config |
| 1740 | self.device_config: DeviceConfig = device_config # type: ignore |
| 1741 | self.load_config: LoadConfig = load_config |
| 1742 | self.quant_config: Optional[QuantConfigBase] = quant_config |
| 1743 | self.graph_opt_config: Optional[GraphOptimizationConfig] = graph_opt_config |
| 1744 | self.early_stop_config: Optional[EarlyStopConfig] = early_stop_config |
| 1745 | self.plas_attention_config: Optional[PlasAttentionConfig] = plas_attention_config |
| 1746 | self.structured_outputs_config: StructuredOutputsConfig = structured_outputs_config |
| 1747 | self.router_config: RouterConfig = router_config |
| 1748 | self.routing_replay_config = routing_replay_config |
| 1749 | self.deploy_modality: DeployModality = deploy_modality if deploy_modality is not None else DeployModality.MIXED |
| 1750 | max_capture_shape = self.scheduler_config.max_num_seqs |
| 1751 | if self.speculative_config is not None and self.speculative_config.method in ["mtp", "suffix"]: |
| 1752 | max_capture_shape = self.scheduler_config.max_num_seqs * ( |
| 1753 | self.speculative_config.num_speculative_tokens + 1 |
| 1754 | ) |
| 1755 | assert max_capture_shape % 2 == 0, "CUDAGraph only supports capturing even token nums in MTP scenarios." |
| 1756 | self.graph_opt_config.real_bsz_to_captured_size = { |
| 1757 | k: 0 for k in range(1, self.scheduler_config.max_num_seqs + 1) |
| 1758 | } |
| 1759 | if self.graph_opt_config.cudagraph_only_prefill: |
| 1760 | max_capture_shape = 512 |
| 1761 | else: |
nothing calls this directly
no test coverage detected