()
| 149 | |
| 150 | |
| 151 | def main() -> None: |
| 152 | split = "holdout" |
| 153 | index_dir = Path("uncommon_route/data/v2_splits") |
| 154 | split_path = index_dir / f"{split}.jsonl" |
| 155 | if not split_path.exists(): |
| 156 | raise SystemExit(f"Missing split: {split_path}") |
| 157 | |
| 158 | rows = load_all_question_bank_rows(split_path) |
| 159 | benchmark_counts = rows_per_benchmark(rows) |
| 160 | |
| 161 | sig_a = MetadataSignal() |
| 162 | sig_b = StructuralSignal() |
| 163 | sig_c = EmbeddingSignal( |
| 164 | index_path=index_dir / "seed_embeddings.npy", |
| 165 | labels_path=index_dir / "seed_labels.json", |
| 166 | model_name="BAAI/bge-small-en-v1.5", |
| 167 | classifier_fallback_threshold=BASELINE_CLASSIFIER_FALLBACK_THRESHOLD, |
| 168 | ) |
| 169 | calibrator = _load_calibrator_if_exists(index_dir) |
| 170 | |
| 171 | if sig_c._embed_fn is None: |
| 172 | raise RuntimeError("Embedding model failed to load; ensure local HF cache is available") |
| 173 | |
| 174 | base_embed = sig_c._embed_fn |
| 175 | embed_cache: dict[str, Any] = {} |
| 176 | |
| 177 | def cached_embed(text: str) -> Any: |
| 178 | if text not in embed_cache: |
| 179 | embed_cache[text] = base_embed(text) |
| 180 | return embed_cache[text] |
| 181 | |
| 182 | sig_c._embed_fn = cached_embed |
| 183 | |
| 184 | print(f"Loaded {len(rows)} holdout rows.") |
| 185 | print("Precomputing Signal A and Signal B votes...") |
| 186 | vote_a_by_id = {row["id"]: sig_a.predict(row) for row in rows} |
| 187 | vote_b_by_id = {row["id"]: sig_b.predict(row) for row in rows} |
| 188 | |
| 189 | fallback_thresholds = [0.90, 0.92, 0.95, 0.97] |
| 190 | vote_c_by_threshold: dict[float, dict[str, Any]] = {} |
| 191 | print("Precomputing Signal C votes by classifier fallback threshold...") |
| 192 | for threshold in fallback_thresholds: |
| 193 | sig_c._clf_fallback_threshold = threshold |
| 194 | vote_c_by_threshold[threshold] = { |
| 195 | row["id"]: sig_c.predict(row) |
| 196 | for row in rows |
| 197 | } |
| 198 | |
| 199 | def run_config(config: ExperimentConfig) -> dict[str, Any]: |
| 200 | original_low_threshold = ensemble_mod._LOW_ESCALATION_THRESHOLD |
| 201 | ensemble_mod._LOW_ESCALATION_THRESHOLD = config.low_escalation_threshold |
| 202 | ensemble_2sig = Ensemble( |
| 203 | weights=[config.weights_a, config.weights_c], |
| 204 | risk_tolerance=0.5, |
| 205 | calibrator=calibrator, |
| 206 | ) |
| 207 | ensemble_3sig = Ensemble( |
| 208 | weights=list(THREE_SIGNAL_WEIGHTS), |
no test coverage detected