Calculates various performance metrics based on the inputs and outputs.
(
input_requests: list[SampleRequest],
outputs: list[RequestFuncOutput],
dur_s: float,
selected_percentiles: list[float],
goodput_config_dict: dict[str, float],
)
| 151 | |
| 152 | |
| 153 | def calculate_metrics( |
| 154 | input_requests: list[SampleRequest], |
| 155 | outputs: list[RequestFuncOutput], |
| 156 | dur_s: float, |
| 157 | selected_percentiles: list[float], |
| 158 | goodput_config_dict: dict[str, float], |
| 159 | ) -> tuple[BenchmarkMetrics, list[int]]: |
| 160 | """Calculates various performance metrics based on the inputs and outputs.""" |
| 161 | input_lens: list[int] = [] |
| 162 | infer_input_lens: list[int] = [] # 推理侧输入token数 |
| 163 | actual_output_lens: list[int] = [] |
| 164 | total_input = 0 |
| 165 | completed = 0 |
| 166 | good_completed = 0 |
| 167 | itls: list[float] = [] |
| 168 | s_itls: list[float] = [] |
| 169 | tpots: list[float] = [] |
| 170 | all_tpots: list[float] = [] |
| 171 | ttfts: list[float] = [] |
| 172 | s_ttfts: list[float] = [] |
| 173 | e2els: list[float] = [] |
| 174 | s_e2els: list[float] = [] |
| 175 | s_decodes: list[float] = [] |
| 176 | for i in range(len(outputs)): |
| 177 | if outputs[i].success: |
| 178 | output_len = outputs[i].output_tokens |
| 179 | |
| 180 | if not output_len: |
| 181 | print("no output_len") |
| 182 | # We use the tokenizer to count the number of output tokens |
| 183 | # for some serving backends instead of looking at |
| 184 | # len(outputs[i].itl) since multiple output tokens may be |
| 185 | # bundled together |
| 186 | # Note : this may inflate the output token count slightly |
| 187 | |
| 188 | actual_output_lens.append(output_len) |
| 189 | input_lens.append(outputs[i].prompt_len) |
| 190 | infer_input_lens.append(outputs[i].prompt_tokens) |
| 191 | total_input += outputs[i].prompt_tokens |
| 192 | tpot = 0 |
| 193 | if output_len > 1: |
| 194 | latency_minus_ttft = outputs[i].latency - outputs[i].ttft |
| 195 | tpot = latency_minus_ttft / (output_len - 1) |
| 196 | tpots.append(tpot) |
| 197 | # Note: if output_len <= 1, we regard tpot as 0 for goodput |
| 198 | all_tpots.append(tpot) |
| 199 | itls += outputs[i].itl |
| 200 | # 推理侧ITL |
| 201 | s_a = outputs[i].arrival_time[1:] |
| 202 | for j in range(len(s_a) - 2): |
| 203 | s_itls.append(s_a[j + 1] - s_a[j]) |
| 204 | ttfts.append(outputs[i].ttft) |
| 205 | # 推理侧TTFT |
| 206 | s_ttfts.append(outputs[i].arrival_time[1]) |
| 207 | e2els.append(outputs[i].latency) |
| 208 | # 推理侧整句时延 |
| 209 | s_e2els.append(outputs[i].arrival_time[-1]) |
| 210 | # 解码速度去掉首token |
no test coverage detected