Inspect traces for correctness. Returns a summary dict with pass/fail counts and per-workload latencies.
(traces: list[dict], problem_name: str)
| 271 | |
| 272 | |
| 273 | def inspect_traces(traces: list[dict], problem_name: str) -> dict: |
| 274 | """Inspect traces for correctness. |
| 275 | |
| 276 | Returns a summary dict with pass/fail counts and per-workload latencies. |
| 277 | """ |
| 278 | total = len(traces) |
| 279 | passed = 0 |
| 280 | failed = 0 |
| 281 | latencies = [] |
| 282 | failure_reasons = [] |
| 283 | |
| 284 | for trace in traces: |
| 285 | evaluation = trace.get("evaluation", {}) |
| 286 | status = evaluation.get("status", "UNKNOWN") |
| 287 | |
| 288 | if status == "PASSED": |
| 289 | passed += 1 |
| 290 | perf = evaluation.get("performance") or {} |
| 291 | latency = perf.get("latency_ms") |
| 292 | if latency is not None: |
| 293 | latencies.append(latency) |
| 294 | else: |
| 295 | failed += 1 |
| 296 | log = evaluation.get("log", "") |
| 297 | failure_reasons.append(f" [{status}] {log[:200]}") |
| 298 | |
| 299 | return { |
| 300 | "problem": problem_name, |
| 301 | "total": total, |
| 302 | "passed": passed, |
| 303 | "failed": failed, |
| 304 | "latencies_ms": latencies, |
| 305 | "failure_reasons": failure_reasons, |
| 306 | } |
| 307 | |
| 308 | |
| 309 | def print_summary(summaries: list[dict]): |