| 416 | |
| 417 | |
| 418 | def generate_report(results: list[dict], naive_sizes: dict[str, int]) -> str: |
| 419 | lines = [ |
| 420 | "# TraceDecay vs Token-Savior Benchmark", |
| 421 | "", |
| 422 | f"Generated: {time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime())}", |
| 423 | "", |
| 424 | "Side-by-side comparison on real-world Python repositories. Both tools index", |
| 425 | "the same clone, and the same random sample of symbols (seed=42) is used for", |
| 426 | "query timing so the find_symbol column is directly comparable.", |
| 427 | "", |
| 428 | "**Memory notes.** token-savior's peak memory is measured with `tracemalloc`", |
| 429 | "(Python heap only). tracedecay runs as a subprocess, so its peak is the", |
| 430 | "`ru_maxrss` delta from `getrusage(RUSAGE_CHILDREN)` (resident set size).", |
| 431 | "These are *not* identical units — treat them as order-of-magnitude.", |
| 432 | "", |
| 433 | "**Query timing.** token-savior is called in-process (pure Python dict", |
| 434 | "lookups). tracedecay is driven over MCP via `tracedecay serve --timings`", |
| 435 | "and the per-query column reports the handler's `_meta.duration_us` —", |
| 436 | "i.e. the time spent inside the Rust handler, with JSON-RPC / stdio /", |
| 437 | "Python-parse overhead stripped out. A warm-up call is issued before", |
| 438 | "each timed loop. `get_change_impact` for tracedecay sums the handler", |
| 439 | "times of `search → impact`, mirroring how an agent must resolve the", |
| 440 | "symbol to a node_id before querying.", |
| 441 | "", |
| 442 | ] |
| 443 | for r in results: |
| 444 | name = r["repo"] |
| 445 | ts = r.get("token_savior") or {} |
| 446 | tk = r.get("tracedecay") or {} |
| 447 | lines += [ |
| 448 | f"## {name}", |
| 449 | "", |
| 450 | f"Naive `.py` source size: {fmt_bytes(naive_sizes.get(name))}", |
| 451 | "", |
| 452 | "| Metric | token-savior | tracedecay |", |
| 453 | "|--------|--------------|-----------|", |
| 454 | f"| Cold index time | {fmt_sec(ts.get('cold_index_seconds'))} | {fmt_sec(tk.get('cold_index_seconds'))} |", |
| 455 | f"| Warm reindex time | {fmt_sec(ts.get('warm_index_seconds'))} | {fmt_sec(tk.get('warm_index_seconds'))} |", |
| 456 | f"| Peak memory (cold) | {fmt_bytes(ts.get('cold_index_peak_memory_bytes'))} | {fmt_bytes(tk.get('cold_index_peak_memory_bytes'))} |", |
| 457 | f"| Cache / DB size | {fmt_bytes(ts.get('cache_size_bytes'))} | {fmt_bytes(tk.get('cache_size_bytes'))} |", |
| 458 | f"| Files indexed | {fmt_int(ts.get('total_files'))} | {fmt_int(tk.get('file_count'))} |", |
| 459 | f"| Symbols / nodes | {fmt_int(ts.get('symbol_table_size'))} | {fmt_int(tk.get('node_count'))} |", |
| 460 | f"| find_symbol avg | {fmt_ms(ts.get('find_symbol_avg_ms'))} | {fmt_ms(tk.get('find_symbol_avg_ms'))} |", |
| 461 | f"| get_function_source avg | {fmt_ms(ts.get('get_function_source_avg_ms'))} | {fmt_ms(tk.get('get_function_source_avg_ms'))} |", |
| 462 | f"| get_change_impact avg | {fmt_ms(ts.get('get_change_impact_avg_ms'))} | {fmt_ms(tk.get('get_change_impact_avg_ms'))} |", |
| 463 | "", |
| 464 | ] |
| 465 | if tk.get("files_by_language"): |
| 466 | langs = ", ".join(f"{k}={v}" for k, v in sorted(tk["files_by_language"].items())) |
| 467 | lines += [f"_tracedecay indexed languages:_ {langs}", ""] |
| 468 | if tk.get("error"): |
| 469 | lines += [f"_tracedecay error:_ `{tk['error'][:300]}`", ""] |
| 470 | return "\n".join(lines) |
| 471 | |
| 472 | |
| 473 | # --------------------------------------------------------------------------- |