()
| 56 | |
| 57 | |
| 58 | def main(): |
| 59 | summary = json.load(open(SUMMARY)) |
| 60 | cfg = json.load(open(CONFIGS))["models"] |
| 61 | by_name = {m["model"]: m for m in summary} |
| 62 | |
| 63 | open_models = [ |
| 64 | m for m in summary |
| 65 | if cfg.get(m["model"], {}).get("type") == "open" |
| 66 | and cfg.get(m["model"], {}).get("params_B") |
| 67 | and cfg[m["model"]]["params_B"] > 0 |
| 68 | and m["model"] not in CALIBRATION_EXCLUDE |
| 69 | ] |
| 70 | log_p = np.array([math.log10(cfg[m["model"]]["params_B"]) for m in open_models]) |
| 71 | |
| 72 | rows = [] |
| 73 | for lam in LAMBDAS: |
| 74 | accs = np.array([acc_at(m["tier_stats"], lam) for m in open_models]) |
| 75 | slope, intercept, r, _, _ = stats.linregress(log_p, accs) |
| 76 | r2 = r ** 2 |
| 77 | resid = accs - (slope * log_p + intercept) |
| 78 | se = math.sqrt(float(np.sum(resid ** 2)) / max(len(accs) - 2, 1)) |
| 79 | pi_factor = 10 ** (1.645 * se / abs(slope)) if slope else float("inf") |
| 80 | # LOO median multiplicative parameter error |
| 81 | folds = [] |
| 82 | n = len(open_models) |
| 83 | for i in range(n): |
| 84 | mask = np.ones(n, bool); mask[i] = False |
| 85 | sl, ic, _, _, _ = stats.linregress(log_p[mask], accs[mask]) |
| 86 | if sl > 0: |
| 87 | pred_log = (accs[i] - ic) / sl |
| 88 | folds.append(10 ** abs(pred_log - log_p[i])) |
| 89 | med_fold = float(np.median(folds)) |
| 90 | within2 = float(np.mean(np.array(folds) <= 2)) |
| 91 | # spotlight estimates |
| 92 | ests = {} |
| 93 | for label, name in SPOTLIGHT: |
| 94 | m = by_name.get(name) |
| 95 | if m and slope > 0: |
| 96 | a = acc_at(m["tier_stats"], lam) |
| 97 | ests[label] = 10 ** ((a - intercept) / slope) |
| 98 | else: |
| 99 | ests[label] = None |
| 100 | rows.append({ |
| 101 | "lambda": lam, |
| 102 | "slope_pp": slope * 100, |
| 103 | "r_squared": r2, |
| 104 | "loo_median_fold": med_fold, |
| 105 | "within_2x": within2, |
| 106 | "pi_factor": pi_factor, |
| 107 | "estimates": ests, |
| 108 | }) |
| 109 | |
| 110 | OUT_JSON.write_text(json.dumps({ |
| 111 | "n_calibration": len(open_models), |
| 112 | "operating_point": 0.0, |
| 113 | "spotlight": [s[0] for s in SPOTLIGHT], |
| 114 | "rows": rows, |
| 115 | }, indent=2)) |
no test coverage detected