(
# E.g., "ttft"
metric_attribute_name: str,
# E.g., "TTFT"
metric_name: str,
# E.g., "Time to First Token"
metric_header: str,
)
| 676 | } |
| 677 | |
| 678 | def process_one_metric( |
| 679 | # E.g., "ttft" |
| 680 | metric_attribute_name: str, |
| 681 | # E.g., "TTFT" |
| 682 | metric_name: str, |
| 683 | # E.g., "Time to First Token" |
| 684 | metric_header: str, |
| 685 | ): |
| 686 | # This function prints and adds statistics of the specified |
| 687 | # metric. |
| 688 | if metric_attribute_name not in selected_percentile_metrics: |
| 689 | return |
| 690 | print("{s:{c}^{n}}".format(s=metric_header, n=50, c="-")) |
| 691 | print( |
| 692 | "{:<40} {:<10.2f}".format( |
| 693 | f"Mean {metric_name} (ms):", |
| 694 | getattr(metrics, f"mean_{metric_attribute_name}_ms"), |
| 695 | ) |
| 696 | ) |
| 697 | print( |
| 698 | "{:<40} {:<10.2f}".format( |
| 699 | f"Median {metric_name} (ms):", |
| 700 | getattr(metrics, f"median_{metric_attribute_name}_ms"), |
| 701 | ) |
| 702 | ) |
| 703 | result[f"mean_{metric_attribute_name}_ms"] = getattr(metrics, f"mean_{metric_attribute_name}_ms") |
| 704 | result[f"median_{metric_attribute_name}_ms"] = getattr(metrics, f"median_{metric_attribute_name}_ms") |
| 705 | result[f"std_{metric_attribute_name}_ms"] = getattr(metrics, f"std_{metric_attribute_name}_ms") |
| 706 | for p, value in getattr(metrics, f"percentiles_{metric_attribute_name}_ms"): |
| 707 | p_word = str(int(p)) if int(p) == p else str(p) |
| 708 | print("{:<40} {:<10.2f}".format(f"P{p_word} {metric_name} (ms):", value)) |
| 709 | result[f"p{p_word}_{metric_attribute_name}_ms"] = value |
| 710 | |
| 711 | def process_pd_metrics(model_outputs, metric_key, is_time=True): |
| 712 | # 收集所有该 metric 的数值 |
no test coverage detected