Method from_config

lmdeploy/pytorch/config.py:578–625

(
        cls,
        method: str,
        num_speculative_tokens: int,
        model: str,
        target_cache_cfg: CacheConfig,
        target_model: str = None,
        dtype: str = 'auto',
        trust_remote_code: bool = False,
        model_format: str = None,
        hf_overrides: dict[str, Any] = None,
        dist_config: DistConfig = None,
    )

    @classmethod
    def from_config(
        cls,
        method: str,
        num_speculative_tokens: int,
        model: str,
        target_cache_cfg: CacheConfig,
        target_model: str = None,
        dtype: str = 'auto',
        trust_remote_code: bool = False,
        model_format: str = None,
        hf_overrides: dict[str, Any] = None,
        dist_config: DistConfig = None,
    ):
        model = model or target_model
        dist_config = dist_config or DistConfig()
        model_config = ModelConfig.from_pretrained(model,
                                                   trust_remote_code=trust_remote_code,
                                                   dtype=dtype,
                                                   dist_config=dist_config,
                                                   is_draft_model=True,
                                                   spec_method=method,
                                                   block_size=target_cache_cfg.block_size,
                                                   model_format=model_format,
                                                   hf_overrides=hf_overrides,
                                                   )
        cache_config = None
        # include medusa
        no_caches = ['medusa']
        if method not in no_caches:
            cache_config = CacheConfig(max_batches=target_cache_cfg.max_batches,
                                       block_size=target_cache_cfg.block_size,
                                       kernel_block_size=target_cache_cfg.kernel_block_size,
                                       num_cpu_blocks=target_cache_cfg.num_cpu_blocks,
                                       num_gpu_blocks=target_cache_cfg.num_gpu_blocks,
                                       cache_max_entry_count=target_cache_cfg.cache_max_entry_count,
                                       max_prefill_token_num=target_cache_cfg.max_prefill_token_num,
                                       device_type=target_cache_cfg.device_type,
                                       quant_policy=target_cache_cfg.quant_policy,
                                       migration_backend=target_cache_cfg.migration_backend)
        obj = cls(
            model=model,
            method=method,
            cache_config=cache_config,
            model_config=model_config,
            dist_config=dist_config,
            num_speculative_tokens=num_speculative_tokens,
        )
        return obj


@dataclass