(rec)
| 120 | is_thinking = model_info.get("thinking", False) |
| 121 | |
| 122 | def patch_one_probe(rec): |
| 123 | pid = rec["probe_id"] |
| 124 | question = rec["question"] |
| 125 | gold = enriched_by_pid.get(pid, {}) |
| 126 | |
| 127 | # Re-query with robust retries |
| 128 | mres = query_model_robust(model_id, question, is_thinking=is_thinking) |
| 129 | response = mres["response"] |
| 130 | if not response.strip(): |
| 131 | return rec # still failed, leave as is |
| 132 | |
| 133 | # Re-judge |
| 134 | gold_for_judge = dict(gold) |
| 135 | gold_for_judge["name"] = gold.get("name") or rec.get("researcher_name", "") |
| 136 | jres = judge_evidence(question, response, gold_for_judge) |
| 137 | verdict = jres["verdict"] |
| 138 | no_cs_match = bool(gold.get("no_cs_match")) |
| 139 | score = score_4way(verdict, lam=-1.0, no_cs_match=no_cs_match) |
| 140 | |
| 141 | new_rec = { |
| 142 | "probe_id": pid, |
| 143 | "tier": rec["tier"], |
| 144 | "source_type": rec.get("source_type", "researcher"), |
| 145 | "domain": rec.get("domain"), |
| 146 | "question": question, |
| 147 | "researcher_name": rec.get("researcher_name"), |
| 148 | "no_cs_match": no_cs_match, |
| 149 | "model_response": (response or "")[:600], |
| 150 | "verdict": verdict, |
| 151 | "score": score, |
| 152 | "judge_raw": jres.get("judge_raw_output"), |
| 153 | "model_query_status": mres["status_code"], |
| 154 | "model_query_error": mres["error"], |
| 155 | } |
| 156 | return new_rec |
| 157 | |
| 158 | patched_results = list(existing["results"]) |
| 159 | pid_to_idx = {r["probe_id"]: i for i, r in enumerate(patched_results)} |
nothing calls this directly
no test coverage detected