Function benchmark_metrics

benchmarks/benchmark_serving.py:831–970

Benchmark metrics statistics，generate benchmark result

(
    benchmark_duration: float,
    result_file: str,
    selected_percentiles: list[float],
    selected_percentile_metrics: list[str],
    goodput_config_dict: dict[str, float],
)

def benchmark_metrics(
    benchmark_duration: float,
    result_file: str,
    selected_percentiles: list[float],
    selected_percentile_metrics: list[str],
    goodput_config_dict: dict[str, float],
):
    """Benchmark metrics statistics，generate benchmark result"""
    outputs = []
    with open(result_file) as f:
        for line in f.readlines():
            if "RequestFuncOutput" in line:
                start = line.find("RequestFuncOutput")
                end = line.rfind(")")
                para_str = line[start : end + 1]

                output = eval(para_str)
                outputs.append(output)

    input_requests = [[]] * len(outputs)
    goodput_config_dict = check_goodput_args(args)

    metrics, actual_output_lens = calculate_metrics(
        # input_requests=input_requests,
        outputs=outputs,
        dur_s=benchmark_duration,
        selected_percentiles=selected_percentiles,
        goodput_config_dict=goodput_config_dict,
    )

    print("{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
    print("{:<40} {:<10}".format("Successful requests:", metrics.completed))
    print("{:<40} {:<10.2f}".format("Benchmark duration (s):", benchmark_duration))
    print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input))
    print("{:<40} {:<10}".format("Total generated tokens:", metrics.total_output))
    print("{:<40} {:<10.2f}".format("Request throughput (req/s):", metrics.request_throughput))
    if goodput_config_dict:
        print("{:<40} {:<10.2f}".format("Request goodput (req/s):", metrics.request_goodput))
    print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):", metrics.output_throughput))
    print("{:<40} {:<10.2f}".format("Total Token throughput (tok/s):", metrics.total_token_throughput))

    result = {
        "duration": benchmark_duration,
        "completed": metrics.completed,
        "total_input_tokens": metrics.total_input,
        "total_output_tokens": metrics.total_output,
        "request_throughput": metrics.request_throughput,
        "request_goodput:": (metrics.request_goodput if goodput_config_dict else None),
        "output_throughput": metrics.output_throughput,
        "total_token_throughput": metrics.total_token_throughput,
        "input_lens": [output.prompt_len for output in outputs],
        "output_lens": actual_output_lens,
        "ttfts": [output.ttft for output in outputs],
        "itls": [output.itl for output in outputs],
        "input_texts": ["" for input in input_requests],
        "generated_texts": [output.generated_text for output in outputs],
        "errors": [output.error for output in outputs],
    }