Class RequestMetrics

fastdeploy/engine/request.py:802–956

Metrics associated with a request. Attributes: arrival_time: The time when the request arrived. preprocess_start_time: The time when the preprocess started. preprocess_end_time: The time when the preprocess ended. scheduler_recv_req_time: The time when the schedu

@dataclass(slots=True)
class RequestMetrics:
    """Metrics associated with a request.

    Attributes:
        arrival_time: The time when the request arrived.
        preprocess_start_time: The time when the preprocess started.
        preprocess_end_time: The time when the preprocess ended.
        scheduler_recv_req_time: The time when the scheduler received the request.
        engine_get_req_time: The time when the engine got the request.
        ask_decode_resource_start_time: The time when the engine asks for decode resource.
        ask_decode_resource_finish_time: The time when the engine has asked for decode resource.
        inference_start_time: The time when engine adds request to the running queue in resource manager.
        wait_for_sending_cache_time: The time when the engine waited for sending cache.
        send_request_output_to_decode_time: The time when the engine sent request_output to decode.
        decode_recv_req_time: The time when the decode received the request.
        decode_preallocate_req_time: The time when the decode has preallocated resource for the request.
        decode_recv_first_token_time: The time when the decode received the first token.
        decode_inference_start_time: The time when the decode sent the request to worker.
        decode_recv_second_token_time: The time when the decode received the second token.

        first_token_time: The cost time between engine_recv_first_token_time and inference_start_time
        time_in_queue: The time the request spent in the queue.
        model_forward_time: The time spent in the model forward pass when this
                            request was in the batch.
        model_execute_time: The time spent in the model execute function. This
                            will include model forward, block/sync across
                            workers, cpu-gpu sync time and sampling time.
        request_start_time: Time to accept the request

    """

    arrival_time: Optional[float] = None  # api server receives request
    preprocess_start_time: Optional[float] = None  # preprocess start time in api server
    preprocess_end_time: Optional[float] = None  # preprocess end time in api server

    scheduler_recv_req_time: Optional[float] = None  # scheduler receives request and add to scheduler
    engine_get_req_time: Optional[float] = None  # engine gets request from scheduler
    ask_decode_resource_start_time: Optional[float] = None  # engine asks decode resource (only valid for prefill)
    ask_decode_resource_finish_time: Optional[float] = None  # engine has got decode resource (only valid for prefill)
    add_req_to_resource_manager_time: Optional[float] = None  # engine adds request to resource manager
    inference_start_time: Optional[float] = None  # requests are added into the engine work queue
    engine_recv_latest_token_time: Optional[float] = None  # receive the latest token from worker
    engine_recv_first_token_time: Optional[float] = None  # receive first token from worker
    wait_for_sending_cache_time: Optional[float] = None  # wait for sending cache (only valid for prefill)
    send_request_output_to_decode_time: Optional[float] = (
        None  # send request_output to worker (only valid for prefill)
    )

    decode_recv_req_time: Optional[float] = None  # decode receive request from prefill (only valid for decode)
    decode_preallocate_req_time: Optional[float] = (
        None  # decode has preallocatee resource for req (only valid for decode)
    )
    decode_recv_first_token_time: Optional[float] = (
        None  # decode receive request_output with first token from prefill (only valid for decode)
    )
    decode_inference_start_time: Optional[float] = (
        None  # decode adds request to the engine work queue (only valid for decode)
    )