Function benchmark

benchmarks/benchmark_serving.py:324–828

Benchmarks an API endpoint using a given set of sample inputs and returns

(
    backend: str,
    api_url: str,
    base_url: str,
    model_id: str,
    model_name: str,
    input_requests: list[SampleRequest],
    hyper_parameters: dict,
    logprobs: Optional[int],
    request_rate: float,
    burstiness: float,
    disable_tqdm: bool,
    profile: bool,
    selected_percentile_metrics: list[str],
    selected_percentiles: list[float],
    ignore_eos: bool,
    debug: bool,
    pd_metrics: bool,
    goodput_config_dict: dict[str, float],
    max_concurrency: Optional[int],
    lora_modules: Optional[Iterable[str]],
    extra_body: Optional[dict],
    ip_list: Optional[list[str]] = None,
)

async def benchmark(
    backend: str,
    api_url: str,
    base_url: str,
    model_id: str,
    model_name: str,
    input_requests: list[SampleRequest],
    hyper_parameters: dict,
    logprobs: Optional[int],
    request_rate: float,
    burstiness: float,
    disable_tqdm: bool,
    profile: bool,
    selected_percentile_metrics: list[str],
    selected_percentiles: list[float],
    ignore_eos: bool,
    debug: bool,
    pd_metrics: bool,
    goodput_config_dict: dict[str, float],
    max_concurrency: Optional[int],
    lora_modules: Optional[Iterable[str]],
    extra_body: Optional[dict],
    ip_list: Optional[list[str]] = None,
):
    """Benchmarks an API endpoint using a given set of sample inputs and returns"""
    if backend in ASYNC_REQUEST_FUNCS:
        request_func = ASYNC_REQUEST_FUNCS[backend]
    else:
        raise ValueError(f"Unknown backend: {backend}")

    print("Starting initial single prompt test run...")
    test_prompt, test_output_len, test_no = (
        input_requests[0].prompt,
        input_requests[0].expected_output_len,
        input_requests[0].no,
    )
    test_history_QA = input_requests[0].history_QA
    response_format = input_requests[0].response_format
    random_flag = input_requests[0].random_flag

    if len(ip_list) >= 1:
        api_url = f"http://{ip_list[0]}{args.endpoint}"

    test_input = RequestFuncInput(
        model=model_id,
        model_name=model_name,
        prompt=test_prompt,
        no=test_no,
        prompt_len=0,
        history_QA=test_history_QA,
        hyper_parameters=hyper_parameters,
        api_url=api_url,
        output_len=test_output_len,
        logprobs=logprobs,
        ignore_eos=ignore_eos,
        debug=debug,
        pd_metrics=pd_metrics,
        extra_body=extra_body,