(evals: list[dict], temperature: float, buckets: int = 10)
| 38 | |
| 39 | |
| 40 | def _compute_ece(evals: list[dict], temperature: float, buckets: int = 10) -> float: |
| 41 | if not evals: |
| 42 | return 0.0 |
| 43 | cal = PlattCalibrator(temperature=temperature) |
| 44 | bucket_data: list[list[tuple[float, float]]] = [[] for _ in range(buckets)] |
| 45 | for item in evals: |
| 46 | conf = cal.calibrate(item["confidence"]) |
| 47 | correct = 1.0 if item["correct"] else 0.0 |
| 48 | idx = min(int(conf * buckets), buckets - 1) |
| 49 | bucket_data[idx].append((conf, correct)) |
| 50 | ece = 0.0 |
| 51 | total = len(evals) |
| 52 | for bucket in bucket_data: |
| 53 | if not bucket: |
| 54 | continue |
| 55 | avg_conf = sum(c for c, _ in bucket) / len(bucket) |
| 56 | avg_acc = sum(a for _, a in bucket) / len(bucket) |
| 57 | ece += abs(avg_conf - avg_acc) * len(bucket) / total |
| 58 | return ece |
| 59 | |
| 60 | |
| 61 | def save_calibrator(cal: PlattCalibrator, path: Path) -> None: |
no test coverage detected