Function test_run_eval_reports_coverage_gaps

tests/test_skill_evaluator.py:232–262

Body alignment must catch the hollow-body case: description fires correctly but the body cannot support the questions it claims.

(tmp_path)

@pytest.mark.asyncio
async def test_run_eval_reports_coverage_gaps(tmp_path):
    """Body alignment must catch the hollow-body case: description fires
    correctly but the body cannot support the questions it claims."""
    skill_dir = _make_skill(tmp_path)
    eval_set = _build_eval_set(3, 3)

    async def perfect_trigger(description, question, *, model):
        match = next(p for p in eval_set if p.question == question)
        return match.expected

    async def hollow_coverage(content, question, *, model):
        # Body claims to support but actually doesn't for the first two
        # trigger prompts.
        if question in {"trig 0", "trig 1"}:
            return "unsupported", "body has no material"
        return "supported", ""

    with (
        patch("openkb.skill.evaluator.grade_one", side_effect=perfect_trigger),
        patch("openkb.skill.evaluator.grade_coverage", side_effect=hollow_coverage),
    ):
        result = await run_eval(skill_dir, model="gpt-4o-mini", eval_set=eval_set)

    # Trigger accuracy is still perfect.
    assert result.passed == 6
    # But coverage catches the hollow shell.
    assert result.trigger_questions == 3
    assert len(result.coverage_misses) == 2
    assert {g.prompt.question for g in result.coverage_misses} == {"trig 0", "trig 1"}
    assert all(g.reason == "body has no material" for g in result.coverage_misses)
    assert result.coverage_rate == pytest.approx(1 / 3)


@pytest.mark.asyncio