()
| 130 | |
| 131 | |
| 132 | def main(): |
| 133 | ap = argparse.ArgumentParser() |
| 134 | ap.add_argument("--turns", type=int, default=5, |
| 135 | help="Number of user turns to replay") |
| 136 | ap.add_argument("--max-tokens", type=int, default=8, |
| 137 | help="max_tokens per response (kept small to bound bench time)") |
| 138 | ap.add_argument("--session", type=Path, default=None, |
| 139 | help="Path to session JSONL; default = most recent under " |
| 140 | f"{SESSION_DIR}") |
| 141 | args = ap.parse_args() |
| 142 | |
| 143 | if not TARGET.exists() or not SERVER_BIN.exists(): |
| 144 | print(f"SKIP: prereqs missing (target={TARGET.exists()} bin={SERVER_BIN.exists()})") |
| 145 | return 0 |
| 146 | |
| 147 | if args.session: |
| 148 | session = args.session |
| 149 | else: |
| 150 | candidates = sorted(SESSION_DIR.glob("*.jsonl"), |
| 151 | key=lambda p: p.stat().st_mtime, reverse=True) |
| 152 | if not candidates: |
| 153 | print(f"No session JSONL under {SESSION_DIR}") |
| 154 | return 1 |
| 155 | session = candidates[0] |
| 156 | print(f"Session: {session.name}", flush=True) |
| 157 | |
| 158 | user_turns = extract_user_turns(session, args.turns) |
| 159 | if len(user_turns) < args.turns: |
| 160 | print(f"Only got {len(user_turns)} turns") |
| 161 | print(f"Extracted {len(user_turns)} user turns:") |
| 162 | for i, t in enumerate(user_turns): |
| 163 | print(f" [{i+1}] {t[:80]!r}{'...' if len(t)>80 else ''}") |
| 164 | |
| 165 | # Cold config: cache disabled (slots=0) → every turn re-prefills full history |
| 166 | cold = run_config("COLD (cache disabled)", port=18290, slots=0, |
| 167 | user_turns=user_turns, max_tokens=args.max_tokens, |
| 168 | log_path=Path("/tmp/bench_cold.log")) |
| 169 | |
| 170 | # Warm config: cache enabled (slots=4) → multi-point inline-snap |
| 171 | warm = run_config("WARM (cache enabled)", port=18291, slots=4, |
| 172 | user_turns=user_turns, max_tokens=args.max_tokens, |
| 173 | log_path=Path("/tmp/bench_warm.log")) |
| 174 | |
| 175 | print("\n=== Per-turn latency ===", flush=True) |
| 176 | print(f"{'turn':>4} {'cold':>8} {'warm':>8} {'speedup':>8}") |
| 177 | total_cold = total_warm = 0.0 |
| 178 | for i, (c, w) in enumerate(zip(cold, warm), start=1): |
| 179 | speedup = (c / w) if (w and w > 0) else float("nan") |
| 180 | print(f"{i:>4} {c:>8.2f} {w:>8.2f} {speedup:>7.2f}x") |
| 181 | total_cold += c; total_warm += w |
| 182 | overall = total_cold / total_warm if total_warm else float("nan") |
| 183 | print(f"\ntotal_cold={total_cold:.2f}s total_warm={total_warm:.2f}s " |
| 184 | f"overall speedup={overall:.2f}x") |
| 185 | |
| 186 | return 0 |
| 187 | |
| 188 | |
| 189 | if __name__ == "__main__": |
no test coverage detected