(tmp_path)
| 204 | |
| 205 | @pytest.mark.asyncio |
| 206 | async def test_run_eval_reports_misses(tmp_path): |
| 207 | skill_dir = _make_skill(tmp_path) |
| 208 | eval_set = _build_eval_set(3, 3) |
| 209 | |
| 210 | # Grader always says "trigger" — so the 3 no-trigger prompts will miss. |
| 211 | async def fake_grade(description, question, *, model): |
| 212 | return "trigger" |
| 213 | |
| 214 | with ( |
| 215 | patch("openkb.skill.evaluator.grade_one", side_effect=fake_grade), |
| 216 | patch("openkb.skill.evaluator.grade_coverage", side_effect=_supported_coverage), |
| 217 | ): |
| 218 | result = await run_eval(skill_dir, model="gpt-4o-mini", eval_set=eval_set) |
| 219 | |
| 220 | assert result.total == 6 |
| 221 | assert result.passed == 3 |
| 222 | assert len(result.misses) == 3 |
| 223 | assert all(m.prompt.expected == "no-trigger" for m in result.misses) |
| 224 | assert all(m.graded == "trigger" for m in result.misses) |
| 225 | assert result.pass_rate == pytest.approx(0.5) |
| 226 | # EvalMiss.label sanity check |
| 227 | assert "no-trigger" in result.misses[0].label |
| 228 | assert "trigger" in result.misses[0].label |
| 229 | |
| 230 | |
| 231 | @pytest.mark.asyncio |
nothing calls this directly
no test coverage detected