Class CacheConfig

lmdeploy/pytorch/config.py:90–120

Config of key value cache.

@dataclass
class CacheConfig:
    """Config of key value cache."""

    max_batches: int
    block_size: int
    num_cpu_blocks: int
    num_gpu_blocks: int
    kernel_block_size: int = -1
    window_size: int = -1
    cache_max_entry_count: float = 0.8
    max_prefill_token_num: int = 8192
    enable_prefix_caching: bool = False
    quant_policy: QuantPolicy = QuantPolicy.NONE
    device_type: str = 'cuda'
    num_state_caches: int = None
    states_shapes: list[tuple] = field(default_factory=list)

    # reserved blocks for dummy inputs, init to 0 for unit test.
    num_reserved_gpu_blocks: int = 0

    # For PD Disaggregation
    role: EngineRole = EngineRole.Hybrid
    migration_backend: MigrationBackend = MigrationBackend.DLSlime

    def __post_init__(self):
        """Post init."""
        if self.window_size > 1 and self.enable_prefix_caching:
            logger.warning('Prefix caching is not available for window attention.')
            self.enable_prefix_caching = False
        if self.kernel_block_size == -1:
            self.kernel_block_size = self.block_size


class TPMode(enum.Enum):