Class containing all metrics that are used in this script
| 49 | |
| 50 | @dataclass |
| 51 | class BenchmarkMetrics: |
| 52 | """Class containing all metrics that are used in this script""" |
| 53 | |
| 54 | completed: int |
| 55 | total_input: int |
| 56 | total_output: int |
| 57 | request_throughput: float |
| 58 | request_goodput: float |
| 59 | output_throughput: float |
| 60 | total_token_throughput: float |
| 61 | mean_s_decode: float |
| 62 | median_s_decode: float |
| 63 | std_s_decode: float |
| 64 | percentiles_s_decode: list[tuple[float, float]] |
| 65 | mean_ttft_ms: float |
| 66 | median_ttft_ms: float |
| 67 | std_ttft_ms: float |
| 68 | percentiles_ttft_ms: list[tuple[float, float]] |
| 69 | mean_s_ttft_ms: float |
| 70 | median_s_ttft_ms: float |
| 71 | std_s_ttft_ms: float |
| 72 | percentiles_s_ttft_ms: list[tuple[float, float]] |
| 73 | mean_tpot_ms: float |
| 74 | median_tpot_ms: float |
| 75 | std_tpot_ms: float |
| 76 | percentiles_tpot_ms: list[tuple[float, float]] |
| 77 | mean_itl_ms: float |
| 78 | median_itl_ms: float |
| 79 | std_itl_ms: float |
| 80 | percentiles_itl_ms: list[tuple[float, float]] |
| 81 | mean_s_itl_ms: float |
| 82 | median_s_itl_ms: float |
| 83 | std_s_itl_ms: float |
| 84 | percentiles_s_itl_ms: list[tuple[float, float]] |
| 85 | # E2EL stands for end-to-end latency per request. |
| 86 | # It is the time taken on the client side from sending |
| 87 | # a request to receiving a complete response. |
| 88 | mean_e2el_ms: float |
| 89 | median_e2el_ms: float |
| 90 | std_e2el_ms: float |
| 91 | percentiles_e2el_ms: list[tuple[float, float]] |
| 92 | mean_s_e2el_ms: float |
| 93 | median_s_e2el_ms: float |
| 94 | std_s_e2el_ms: float |
| 95 | percentiles_s_e2el_ms: list[tuple[float, float]] |
| 96 | mean_input_len: float |
| 97 | median_input_len: float |
| 98 | std_input_len: float |
| 99 | percentiles_input_len: list[tuple[float, float]] |
| 100 | mean_s_input_len: float |
| 101 | median_s_input_len: float |
| 102 | std_s_input_len: float |
| 103 | percentiles_s_input_len: list[tuple[float, float]] |
| 104 | mean_output_len: float |
| 105 | median_output_len: float |
| 106 | std_output_len: float |
| 107 | percentiles_output_len: list[tuple[float, float]] |
| 108 |