(records)
| 25 | return (not (r.get("model_response") or "").strip()) and r.get("verdict") == "REFUSAL" |
| 26 | |
| 27 | def score(records): |
| 28 | ts = {t: {"total":0,"correct":0,"wrong":0,"refusal":0} for t in TIERS} |
| 29 | n_excluded = 0 |
| 30 | for r in records: |
| 31 | if r.get("probe_id") not in CLEAN: |
| 32 | continue |
| 33 | if is_error(r): |
| 34 | n_excluded += 1 |
| 35 | continue |
| 36 | t = r.get("tier") |
| 37 | if t not in ts: continue |
| 38 | ts[t]["total"] += 1 |
| 39 | if r.get("refusal"): ts[t]["refusal"] += 1 |
| 40 | elif r.get("correct"): ts[t]["correct"] += 1 |
| 41 | else: ts[t]["wrong"] += 1 |
| 42 | tacc = {} |
| 43 | for t in TIERS: |
| 44 | s = ts[t] |
| 45 | s["score"] = (s["correct"] / s["total"]) if s["total"] else 0.0 # lambda=0 |
| 46 | tacc[t] = s["score"] |
| 47 | acc = sum(tacc.values()) / 7 |
| 48 | tot = sum(s["total"] for s in ts.values()); corr = sum(s["correct"] for s in ts.values()) |
| 49 | return ts, tacc, acc, (corr/tot if tot else 0.0), corr, tot, n_excluded |
| 50 | |
| 51 | summary = []; tot_excluded = 0 |
| 52 | for f in sorted(glob.glob(str(RES / "*.json"))): |
no test coverage detected