(
dataset: list[TestCase],
config: ScoringConfig,
*,
view: str,
)
| 89 | |
| 90 | |
| 91 | def evaluate_dataset( |
| 92 | dataset: list[TestCase], |
| 93 | config: ScoringConfig, |
| 94 | *, |
| 95 | view: str, |
| 96 | ) -> list[dict]: |
| 97 | if view not in {"classifier", "route"}: |
| 98 | raise ValueError(f"Unsupported benchmark view: {view}") |
| 99 | |
| 100 | routing_config = replace(DEFAULT_CONFIG, scoring=config) |
| 101 | results: list[dict] = [] |
| 102 | |
| 103 | for tc in dataset: |
| 104 | expected = expected_tier_for_view(tc, view) |
| 105 | features = build_routing_features(tc.routing_features) |
| 106 | |
| 107 | if view == "classifier": |
| 108 | result = classify(tc.prompt, tc.system_prompt, config) |
| 109 | actual = collapse_tier(result.tier, default="") if result.tier is not None else None |
| 110 | resolved = collapse_tier(result.tier, default="MEDIUM") |
| 111 | confidence = result.confidence |
| 112 | score = 0.0 |
| 113 | else: |
| 114 | decision = route( |
| 115 | tc.prompt, |
| 116 | tc.system_prompt, |
| 117 | config=routing_config, |
| 118 | routing_features=features, |
| 119 | ) |
| 120 | actual = collapse_tier(decision.tier) |
| 121 | resolved = actual |
| 122 | confidence = decision.confidence |
| 123 | score = None |
| 124 | |
| 125 | results.append({ |
| 126 | "expected": expected, |
| 127 | "actual": actual, |
| 128 | "resolved": resolved, |
| 129 | "correct": resolved == expected, |
| 130 | "score": score, |
| 131 | "confidence": max(0.0, min(1.0, float(confidence))), |
| 132 | "category": tc.category, |
| 133 | "lang": tc.lang, |
| 134 | "feature_annotated": features is not None, |
| 135 | "feature_tags": list(feature_slice_tags(features)), |
| 136 | "routing_features": serialize_routing_features(features), |
| 137 | }) |
| 138 | |
| 139 | return results |
| 140 | |
| 141 | |
| 142 | def compute_metrics(evals: list[dict]) -> dict: |
nothing calls this directly
no test coverage detected