Class EngineOutput

lmdeploy/messages.py:657–676

Engine output from turbomind/pytorch engine. Args: status: the response type. token_ids: the newly generated token ids in each iteration. logprobs: the top logprobs for each output position. cache_block_ids: send cache blocks back for migration in

@dataclass
class EngineOutput:
    """Engine output from turbomind/pytorch engine.

    Args:
        status: the response type.
        token_ids: the newly generated token ids in each iteration.
        logprobs: the top logprobs for each output
            position.
        cache_block_ids: send cache blocks back for migration in
            Disaggregated LLM Serving when Prefill Engine is Done.
        req_metrics: request metrics information
    """
    status: ResponseType
    token_ids: list[int]
    logprobs: list[dict[int, float]] = None
    logits: torch.Tensor = None
    last_hidden_state: torch.Tensor = None
    cache_block_ids: list[int] | None = None
    req_metrics: RequestMetrics | None = None
    routed_experts: torch.Tensor = None


@dataclass