| 194 | } |
| 195 | |
| 196 | void print_result(const BenchmarkResults_t& results) { |
| 197 | // Calculate number of stages (1k, 2k, 4k, ...) |
| 198 | int stages; |
| 199 | stages = results.decoding_speed.size(); |
| 200 | |
| 201 | header_print("FLM", "=== Benchmark Results ==="); |
| 202 | std::cout << "\n"; |
| 203 | |
| 204 | // Print table header |
| 205 | std::cout << std::setw(15) << "Context Length" << " | " |
| 206 | << std::setw(21) << "TTFT (s)" << " | " |
| 207 | << std::setw(26) << "Prefill Speed (tok/s)" << " | " |
| 208 | << std::setw(26) << "Decoding Speed (tok/s)" << "\n"; |
| 209 | std::cout << std::string(100, '-') << "\n"; |
| 210 | |
| 211 | // Print results for each stage |
| 212 | for (int i = 0; i < stages && i < results.decoding_speed.size(); i++) { |
| 213 | int context_len = 1 << i; // 1k, 2k, 4k, 8k, etc. |
| 214 | |
| 215 | std::cout << std::setw(14) << context_len << "k" << " | "; |
| 216 | |
| 217 | // TTFT |
| 218 | if (i < results.TTFT.size()) { |
| 219 | std::cout << std::setw(11) << std::fixed << std::setprecision(3) << results.TTFT[i].average |
| 220 | << " ± " << std::setw(7) << std::fixed << std::setprecision(3) << results.TTFT[i].std_variance; |
| 221 | } else { |
| 222 | std::cout << std::setw(21) << "N/A"; |
| 223 | } |
| 224 | std::cout << " | "; |
| 225 | |
| 226 | // Prefill Speed |
| 227 | if (i < results.prefill_speed.size()) { |
| 228 | std::cout << std::setw(14) << std::fixed << std::setprecision(2) << results.prefill_speed[i].average |
| 229 | << " ± " << std::setw(9) << std::fixed << std::setprecision(2) << results.prefill_speed[i].std_variance; |
| 230 | } else { |
| 231 | std::cout << std::setw(26) << "N/A"; |
| 232 | } |
| 233 | std::cout << " | "; |
| 234 | |
| 235 | // Decoding Speed |
| 236 | if (i < results.decoding_speed.size()) { |
| 237 | std::cout << std::setw(11) << std::fixed << std::setprecision(2) << results.decoding_speed[i].average |
| 238 | << " ± " << std::setw(9) << std::fixed << std::setprecision(2) << results.decoding_speed[i].std_variance; |
| 239 | } else { |
| 240 | std::cout << std::setw(26) << "N/A"; |
| 241 | } |
| 242 | std::cout << "\n"; |
| 243 | } |
| 244 | |
| 245 | std::cout << std::string(100, '-') << "\n"; |
| 246 | std::cout << "\n"; |
| 247 | } |
| 248 | |
| 249 | BenchmarkResults_t run_benchmarks(std::string model_tag, std::string bench_config_file, model_list& availble_models){ |
| 250 | BenchmarkResults_t results; |