()
| 22 | |
| 23 | |
| 24 | def main(): |
| 25 | index_dir = Path("uncommon_route/data/v2_splits") |
| 26 | cal_path = index_dir / "calibration.jsonl" |
| 27 | if not cal_path.exists(): |
| 28 | print(f"ERROR: {cal_path} not found.") |
| 29 | sys.exit(1) |
| 30 | |
| 31 | rows = load_all_question_bank_rows(cal_path) |
| 32 | print(f"Calibration rows: {len(rows)}") |
| 33 | |
| 34 | sig_a = MetadataSignal() |
| 35 | sig_b = StructuralSignal() |
| 36 | sig_c = EmbeddingSignal( |
| 37 | index_path=index_dir / "seed_embeddings.npy", |
| 38 | labels_path=index_dir / "seed_labels.json", |
| 39 | model_name="BAAI/bge-small-en-v1.5", |
| 40 | ) |
| 41 | ensemble = Ensemble(weights=[0.50, 0.10, 0.40], risk_tolerance=0.5) |
| 42 | |
| 43 | evals = [] |
| 44 | for row in rows: |
| 45 | vote_a = sig_a.predict(row) |
| 46 | vote_b = sig_b.predict(row) |
| 47 | vote_c = sig_c.predict(row) |
| 48 | result = ensemble.decide([vote_a, vote_b, vote_c]) |
| 49 | if result.tier_id is not None: |
| 50 | correct = result.tier_id == row["target_tier_id"] |
| 51 | evals.append({"confidence": result.confidence, "correct": correct}) |
| 52 | |
| 53 | print(f"Evaluable: {len(evals)}") |
| 54 | calibrator = fit_platt_from_evals(evals) |
| 55 | print(f"Optimal temperature: {calibrator.temperature}") |
| 56 | |
| 57 | out_path = index_dir / "calibration_params.json" |
| 58 | save_calibrator(calibrator, out_path) |
| 59 | print(f"Saved to {out_path}") |
| 60 | |
| 61 | |
| 62 | if __name__ == "__main__": |
no test coverage detected