总结结果
(benchmark_name: str)
| 1278 | |
| 1279 | |
| 1280 | def summarize_results(benchmark_name: str): |
| 1281 | """总结结果""" |
| 1282 | from collections import Counter |
| 1283 | |
| 1284 | results_path = os.path.join( |
| 1285 | config.workdir, "benchmark", benchmark_name, "results.jsonl" |
| 1286 | ) |
| 1287 | |
| 1288 | if not os.path.exists(results_path): |
| 1289 | logger.warning(f"⚠️ Results file not found: {results_path}") |
| 1290 | return |
| 1291 | |
| 1292 | total = 0 |
| 1293 | score_1_cnt = 0 |
| 1294 | pred_counter = Counter() |
| 1295 | round_counter = Counter() |
| 1296 | |
| 1297 | with open(results_path, "r", encoding="utf-8") as f: |
| 1298 | for line in f: |
| 1299 | if not line.strip(): |
| 1300 | continue |
| 1301 | try: |
| 1302 | data = json.loads(line) |
| 1303 | except json.JSONDecodeError: |
| 1304 | continue |
| 1305 | |
| 1306 | total += 1 |
| 1307 | if data.get("score") == 1.0: |
| 1308 | score_1_cnt += 1 |
| 1309 | |
| 1310 | pred = data.get("prediction", "").strip() |
| 1311 | if pred: |
| 1312 | pred_counter[pred] += 1 |
| 1313 | |
| 1314 | final_round = data.get("metrics", {}).get("final_round", 0) |
| 1315 | round_counter[final_round] += 1 |
| 1316 | |
| 1317 | print("\n" + "=" * 60) |
| 1318 | print(f"📊 Benchmark Summary: {benchmark_name}") |
| 1319 | print(f"📊 Model: {TARGET_MODEL}") |
| 1320 | print(f"📊 Language: {TARGET_LANGUAGE}") |
| 1321 | print("=" * 60) |
| 1322 | print(f"Total tasks: {total}") |
| 1323 | print(f"Accepted (score=1.0): {score_1_cnt}") |
| 1324 | print(f"Accuracy: {score_1_cnt / total * 100:.2f}%") |
| 1325 | print("=" * 60) |
| 1326 | print("Result Distribution:") |
| 1327 | for key in ["Accepted", "Wrong Answer", "Time Limit Exceeded", "Runtime Error", |
| 1328 | "Memory Limit Exceeded", "Compile Error", "Timeout"]: |
| 1329 | print(f" {key}: {pred_counter.get(key, 0)}") |
| 1330 | print("=" * 60) |
| 1331 | print("Rounds Distribution:") |
| 1332 | for r in sorted(round_counter.keys()): |
| 1333 | print(f" Round {r}: {round_counter[r]} tasks") |
| 1334 | print("=" * 60) |
| 1335 | |
| 1336 | |
| 1337 | async def main(): |