()
| 153 | |
| 154 | |
| 155 | def main(): |
| 156 | parser = argparse.ArgumentParser(description="v2 evaluation") |
| 157 | parser.add_argument("--split", choices=["holdout", "calibration", "train"], default="holdout") |
| 158 | parser.add_argument("--signals", type=int, choices=[2, 3], default=2, help="2=A+C (default), 3=A+B+C (force B active)") |
| 159 | parser.add_argument("--shadow", action="store_true", default=True, help="Enable Signal B shadow mode (default: on)") |
| 160 | parser.add_argument("--no-shadow", dest="shadow", action="store_false", help="Disable Signal B shadow mode") |
| 161 | parser.add_argument("--risk-tolerance", type=float, default=0.5) |
| 162 | parser.add_argument("--index-dir", default="uncommon_route/data/v2_splits") |
| 163 | parser.add_argument("--verbose", action="store_true") |
| 164 | args = parser.parse_args() |
| 165 | |
| 166 | index_dir = Path(args.index_dir) |
| 167 | split_path = index_dir / f"{args.split}.jsonl" |
| 168 | if not split_path.exists(): |
| 169 | print(f"ERROR: {split_path} not found. Run split_data.py first.") |
| 170 | sys.exit(1) |
| 171 | |
| 172 | rows = load_all_question_bank_rows(split_path) |
| 173 | predict_fn, shadow_tracker = make_v2_predictor( |
| 174 | args.risk_tolerance, index_dir, signals=args.signals, |
| 175 | shadow=(args.shadow and args.signals == 2), |
| 176 | ) |
| 177 | predictor = FunctionPredictor(predict_fn) |
| 178 | mode_label = f"{args.signals}sig" |
| 179 | if args.signals == 2 and args.shadow: |
| 180 | mode_label += "+cond_b" |
| 181 | label = f"v2_{mode_label}_rt{args.risk_tolerance}" |
| 182 | |
| 183 | progress = (lambda msg: print(msg, file=sys.stderr)) if args.verbose else None |
| 184 | |
| 185 | per_row, errors, correct = evaluate_question_bank_rows( |
| 186 | predictor, rows, predictor_label=label, progress=progress, |
| 187 | ) |
| 188 | |
| 189 | bc = rows_per_benchmark(rows) |
| 190 | summary = build_eval_summary( |
| 191 | per_row=per_row, errors=errors, correct=correct, |
| 192 | predictor_label=label, shard=split_path, |
| 193 | sample_mode=f"{args.split}_split", seed=42, |
| 194 | proportional_quotas=None, benchmark_counts=bc, |
| 195 | ) |
| 196 | |
| 197 | s11 = summary["section_11"] |
| 198 | acct = summary["router_accounting"] |
| 199 | |
| 200 | print() |
| 201 | print("=" * 64) |
| 202 | print(f" UncommonRoute v2 ({mode_label}) — {args.split} evaluation") |
| 203 | print("=" * 64) |
| 204 | print(f" risk_tolerance: {args.risk_tolerance}") |
| 205 | print(f" Samples: {summary['sampled']}") |
| 206 | print("-" * 64) |
| 207 | print(f" Tier Match Accuracy : {summary['tier_match_accuracy']:>6.1%}") |
| 208 | print(f" Pass Rate : {s11['pass_rate']:>6.1%}") |
| 209 | print(f" Cost Savings Score : {fmt(s11['cost_savings_score'])}") |
| 210 | print(f" Overall Score : {fmt(acct['overall_score_percent'])}") |
| 211 | print(f" API Errors : {summary['api_errors']}") |
| 212 |
no test coverage detected