Parse engine stats grouped by benchmark run. Uses "Starting main benchmark run..." as the start delimiter and "Serving Benchmark Result" as the end delimiter for each run.
(path: Path)
| 44 | |
| 45 | |
| 46 | def parse_sweep_log(path: Path) -> list[BenchRun]: |
| 47 | """Parse engine stats grouped by benchmark run. |
| 48 | |
| 49 | Uses "Starting main benchmark run..." as the start delimiter and |
| 50 | "Serving Benchmark Result" as the end delimiter for each run. |
| 51 | """ |
| 52 | runs: list[BenchRun] = [] |
| 53 | current: BenchRun | None = None |
| 54 | pending_concurrency = 0 |
| 55 | pending_run = 0 |
| 56 | |
| 57 | for line in path.read_text().splitlines(): |
| 58 | # Track concurrency and run number from lines that precede the |
| 59 | # benchmark start. |
| 60 | m = _RUN_NUMBER_RE.search(line) |
| 61 | if m: |
| 62 | pending_run = int(m.group(1)) |
| 63 | |
| 64 | if "Namespace(" in line: |
| 65 | m = _NAMESPACE_CONC_RE.search(line) |
| 66 | if m: |
| 67 | pending_concurrency = int(m.group(1)) |
| 68 | |
| 69 | if "Starting main benchmark run..." in line: |
| 70 | current = BenchRun( |
| 71 | max_concurrency=pending_concurrency, |
| 72 | run_number=pending_run, |
| 73 | ) |
| 74 | continue |
| 75 | |
| 76 | if "Serving Benchmark Result" in line: |
| 77 | if current: |
| 78 | runs.append(current) |
| 79 | current = None |
| 80 | continue |
| 81 | |
| 82 | if current is None: |
| 83 | continue |
| 84 | |
| 85 | m = _ENGINE_RE.search(line) |
| 86 | if m: |
| 87 | current.stats.append( |
| 88 | { |
| 89 | "running": int(m.group(1)), |
| 90 | "waiting": int(m.group(2)), |
| 91 | "kv_cache_pct": float(m.group(3)), |
| 92 | } |
| 93 | ) |
| 94 | |
| 95 | return runs |
| 96 | |
| 97 | |
| 98 | def summarize(runs: list[BenchRun]) -> pd.DataFrame: |