Patch failed probes in the existing v2 result file.
(model_name, model_info, all_probes_by_pid, enriched_by_pid)
| 101 | |
| 102 | |
| 103 | def patch_one(model_name, model_info, all_probes_by_pid, enriched_by_pid): |
| 104 | """Patch failed probes in the existing v2 result file.""" |
| 105 | out_file = OUT_DIR / f"{model_name}.json" |
| 106 | if not out_file.exists(): |
| 107 | print(f" [SKIP] {model_name}: no v2 result file") |
| 108 | return |
| 109 | with open(out_file) as f: |
| 110 | existing = json.load(f) |
| 111 | |
| 112 | failed = [r for r in existing["results"] |
| 113 | if r.get("model_query_status") != 200 |
| 114 | or r.get("model_query_error")] |
| 115 | print(f" {model_name}: {len(failed)} failed probes to patch") |
| 116 | if not failed: |
| 117 | return |
| 118 | |
| 119 | model_id = model_info["id"] |
| 120 | is_thinking = model_info.get("thinking", False) |
| 121 | |
| 122 | def patch_one_probe(rec): |
| 123 | pid = rec["probe_id"] |
| 124 | question = rec["question"] |
| 125 | gold = enriched_by_pid.get(pid, {}) |
| 126 | |
| 127 | # Re-query with robust retries |
| 128 | mres = query_model_robust(model_id, question, is_thinking=is_thinking) |
| 129 | response = mres["response"] |
| 130 | if not response.strip(): |
| 131 | return rec # still failed, leave as is |
| 132 | |
| 133 | # Re-judge |
| 134 | gold_for_judge = dict(gold) |
| 135 | gold_for_judge["name"] = gold.get("name") or rec.get("researcher_name", "") |
| 136 | jres = judge_evidence(question, response, gold_for_judge) |
| 137 | verdict = jres["verdict"] |
| 138 | no_cs_match = bool(gold.get("no_cs_match")) |
| 139 | score = score_4way(verdict, lam=-1.0, no_cs_match=no_cs_match) |
| 140 | |
| 141 | new_rec = { |
| 142 | "probe_id": pid, |
| 143 | "tier": rec["tier"], |
| 144 | "source_type": rec.get("source_type", "researcher"), |
| 145 | "domain": rec.get("domain"), |
| 146 | "question": question, |
| 147 | "researcher_name": rec.get("researcher_name"), |
| 148 | "no_cs_match": no_cs_match, |
| 149 | "model_response": (response or "")[:600], |
| 150 | "verdict": verdict, |
| 151 | "score": score, |
| 152 | "judge_raw": jres.get("judge_raw_output"), |
| 153 | "model_query_status": mres["status_code"], |
| 154 | "model_query_error": mres["error"], |
| 155 | } |
| 156 | return new_rec |
| 157 | |
| 158 | patched_results = list(existing["results"]) |
| 159 | pid_to_idx = {r["probe_id"]: i for i, r in enumerate(patched_results)} |
| 160 |