()
| 117 | |
| 118 | |
| 119 | def main() -> int: |
| 120 | ap = argparse.ArgumentParser() |
| 121 | ap.add_argument("--root", default="manpages/ubuntu/26.04") |
| 122 | ap.add_argument("--left-model", default="codex/gpt-5.4-mini/medium") |
| 123 | ap.add_argument("--right-model", default="codex/gpt-5.4/medium") |
| 124 | ap.add_argument("--n-per-bucket", type=int, default=50) |
| 125 | ap.add_argument("--jobs", type=int, default=8) |
| 126 | ap.add_argument("--seed", type=int, default=42) |
| 127 | ap.add_argument("--out", default="logs/eval_size_routing.json") |
| 128 | args = ap.parse_args() |
| 129 | |
| 130 | logging.basicConfig(level=logging.WARNING, format="%(message)s") |
| 131 | logging.getLogger("explainshell").setLevel(logging.WARNING) |
| 132 | |
| 133 | sample = sample_files(args.root, args.n_per_bucket, args.seed) |
| 134 | files = [p for _, p in sample] |
| 135 | print(f"sampled {len(files)} files: ", end="") |
| 136 | by_b: dict[tuple[int, int], int] = defaultdict(int) |
| 137 | for b, _ in sample: |
| 138 | by_b[b] += 1 |
| 139 | print(", ".join(f"{lo}-{hi}={by_b[(lo, hi)]}" for lo, hi in BUCKETS)) |
| 140 | |
| 141 | run_dir = os.path.join("logs", f"eval_size_routing_{int(time.time())}") |
| 142 | os.makedirs(run_dir, exist_ok=True) |
| 143 | |
| 144 | left_res = run_extractor("LEFT", args.left_model, run_dir, files, args.jobs) |
| 145 | right_res = run_extractor("RIGHT", args.right_model, run_dir, files, args.jobs) |
| 146 | |
| 147 | rows: list[dict[str, Any]] = [] |
| 148 | skipped = 0 |
| 149 | for bucket, path in sample: |
| 150 | left = left_res.get(path) |
| 151 | right = right_res.get(path) |
| 152 | if ( |
| 153 | not left |
| 154 | or not right |
| 155 | or left.outcome != ExtractionOutcome.SUCCESS |
| 156 | or right.outcome != ExtractionOutcome.SUCCESS |
| 157 | ): |
| 158 | skipped += 1 |
| 159 | continue |
| 160 | d = diff_options(left.mp.options, right.mp.options) |
| 161 | d["bucket"] = list(bucket) |
| 162 | d["path"] = path |
| 163 | d["size"] = os.path.getsize(path) |
| 164 | rows.append(d) |
| 165 | |
| 166 | os.makedirs(os.path.dirname(args.out), exist_ok=True) |
| 167 | with open(args.out, "w") as f: |
| 168 | json.dump( |
| 169 | { |
| 170 | "left_model": args.left_model, |
| 171 | "right_model": args.right_model, |
| 172 | "n_per_bucket": args.n_per_bucket, |
| 173 | "seed": args.seed, |
| 174 | "rows": rows, |
| 175 | "skipped": skipped, |
| 176 | }, |
no test coverage detected