Compute paired speedup ratios between baseline and FMMS runs. Comparisons are made only within the same trial directory (e.g. Qwen3-8B-trial1/baseline vs Qwen3-8B-trial1/fmms-triton), never across trials. Within a trial, each run_number is paired 1:1 (run 0 baseline vs run 0 FMMS, e
(model_dirs: list[Path], max_concurrency: int)
| 89 | ]: |
| 90 | df = load_variant(model_dir, variant_key) |
| 91 | if df is None: |
| 92 | continue |
| 93 | df["variant"] = display_name |
| 94 | df["model"] = model |
| 95 | frames.append( |
| 96 | df[["model", "variant", "max_concurrency", "median_tpot_ms", "run_number"]] |
| 97 | ) |
| 98 | return pd.concat(frames, ignore_index=True) |
| 99 | |
| 100 | |
| 101 | def paired_speedups( |
| 102 | model_dirs: list[Path], |
| 103 | candidate_dir: str, |
| 104 | max_concurrency: int, |
| 105 | ) -> pd.DataFrame: |
| 106 | """Compute paired speedup ratios between baseline and candidate runs. |
| 107 | |
| 108 | Comparisons are made only within the same trial directory (e.g. |
| 109 | Qwen3-8B-trial1/baseline vs Qwen3-8B-trial1/fmms-triton), never across |
| 110 | trials. Within a trial, each run_number is paired 1:1 (run 0 baseline vs |
| 111 | run 0 FMMS, etc.). The median across all paired speedups from all trials |
| 112 | is reported. |
| 113 | """ |
| 114 | frames = [] |
| 115 | for model_dir in model_dirs: |
| 116 | baseline_df = load_variant(model_dir, "baseline") |
| 117 | candidate_df = load_variant(model_dir, candidate_dir) |
| 118 | if baseline_df is None or candidate_df is None: |
| 119 | continue |
| 120 | merged = ( |
| 121 | baseline_df[["max_concurrency", "run_number", "median_tpot_ms"]] |
| 122 | .merge( |
no test coverage detected