(dataset: list[TestCase])
| 32 | |
| 33 | |
| 34 | def _evaluate(dataset: list[TestCase]) -> list[dict]: |
| 35 | cfg = ScoringConfig() |
| 36 | results: list[dict] = [] |
| 37 | for tc in dataset: |
| 38 | result = classify(tc.prompt, tc.system_prompt, cfg) |
| 39 | resolved = _collapse_tier(result.tier.value if result.tier else "MEDIUM") |
| 40 | results.append({ |
| 41 | "expected": _collapse_tier(tc.expected_tier), |
| 42 | "resolved": resolved, |
| 43 | "correct": resolved == _collapse_tier(tc.expected_tier), |
| 44 | "confidence": result.confidence, |
| 45 | "lang": tc.lang, |
| 46 | "category": tc.category, |
| 47 | }) |
| 48 | return results |
| 49 | |
| 50 | |
| 51 | def _tier_f1(evals: list[dict]) -> dict[str, float]: |
no test coverage detected