| 37 | |
| 38 | |
| 39 | def print_summary(results_dir: Path): |
| 40 | raw_frames = {} |
| 41 | num_runs = None |
| 42 | for variant_key, display_name in VARIANTS: |
| 43 | variant_dir = results_dir / variant_key |
| 44 | if not variant_dir.exists(): |
| 45 | print(f"Warning: {variant_dir} not found, skipping") |
| 46 | continue |
| 47 | df, run_name = read_summary(results_dir, variant_key) |
| 48 | print(f"{variant_key}: {run_name}") |
| 49 | num_runs = df["run_number"].nunique() |
| 50 | raw_frames[variant_key] = df |
| 51 | |
| 52 | # Median TPOT across all runs per concurrency level |
| 53 | tpot_frames = {} |
| 54 | for variant_key, display_name in VARIANTS: |
| 55 | if variant_key not in raw_frames: |
| 56 | continue |
| 57 | df = raw_frames[variant_key] |
| 58 | median_tpot = df.groupby("max_concurrency")["median_tpot_ms"].median().rename(display_name) |
| 59 | tpot_frames[variant_key] = median_tpot |
| 60 | |
| 61 | tpot = pd.concat(tpot_frames.values(), axis=1) |
| 62 | |
| 63 | if "Baseline" not in tpot.columns: |
| 64 | print("\nNo baseline found, printing available variants only.\n") |
| 65 | print(tabulate(tpot, headers="keys", tablefmt="grid", floatfmt=".2f")) |
| 66 | return |
| 67 | |
| 68 | baseline = tpot["Baseline"] |
| 69 | |
| 70 | print(f"\nMedian across {num_runs} runs. Speedup: Hodges-Lehmann estimator.\n") |
| 71 | |
| 72 | # Build result with TPOT and Speedup columns interleaved |
| 73 | result = pd.DataFrame(index=tpot.index) |
| 74 | result["Baseline"] = baseline |
| 75 | for variant_key, display_name in VARIANTS[1:]: # skip baseline |
| 76 | if display_name not in tpot.columns: |
| 77 | continue |
| 78 | result[display_name] = tpot[display_name] |
| 79 | # Paired speedup: match run_number 1:1 between baseline and FMMS |
| 80 | baseline_df = raw_frames["baseline"] |
| 81 | fmms_df = raw_frames[variant_key] |
| 82 | merged = baseline_df[["max_concurrency", "run_number", "median_tpot_ms"]].merge( |
| 83 | fmms_df[["max_concurrency", "run_number", "median_tpot_ms"]], |
| 84 | on=["max_concurrency", "run_number"], |
| 85 | suffixes=("_base", "_fmms"), |
| 86 | ) |
| 87 | merged["speedup_pct"] = ( |
| 88 | merged["median_tpot_ms_fmms"] / merged["median_tpot_ms_base"] - 1 |
| 89 | ) * 100 |
| 90 | speedup_stats = merged.groupby("max_concurrency")["speedup_pct"].agg(["median", "std"]) |
| 91 | result[f"{display_name} Speedup"] = speedup_stats.apply( |
| 92 | lambda r: f"{r['median']:+.1f}% ± {r['std']:.1f}" |
| 93 | if pd.notna(r["std"]) |
| 94 | else f"{r['median']:+.1f}%", |
| 95 | axis=1, |
| 96 | ) |