Function grade_coverage

openkb/skill/evaluator.py:321–386

Ask the alignment grader whether the SKILL.md body + references actually contain enough substance to answer the question. This is the orthogonal check to :func:`grade_one`. A skill can have a perfectly-firing description and still be a hollow shell — this catches that. Returns ``"su

(
    skill_content: str,
    question: str,
    *,
    model: str,
)

async def grade_coverage(
    skill_content: str,
    question: str,
    *,
    model: str,
) -> tuple[Literal["supported", "unsupported", "ambiguous"], str]:
    """Ask the alignment grader whether the SKILL.md body + references
    actually contain enough substance to answer the question.

    This is the orthogonal check to :func:`grade_one`. A skill can have a
    perfectly-firing description and still be a hollow shell — this catches
    that. Returns ``"supported"``, ``"unsupported"``, or ``"ambiguous"``
    (parser couldn't extract a verdict from the grader's output) plus a
    one-line reason. Callers should NOT collapse ``"ambiguous"`` into
    ``"unsupported"`` — see :class:`EvalResult.coverage_ambiguous`.
    """
    instructions = (
        "You are auditing a skill for content quality. You will be given "
        "the skill's body (SKILL.md without frontmatter) and any "
        "reference excerpts, plus a user question that the skill's "
        "description claims to handle. Decide whether the body has "
        "substantive material to answer the question.\n\n"
        "Answer with EXACTLY this two-line shape:\n"
        "VERDICT: SUPPORTED  (or UNSUPPORTED)\n"
        "REASON: <one short sentence>\n\n"
        f"{skill_content}"
    )
    agent = Agent(
        name="coverage-grader",
        instructions=instructions,
        model=f"litellm/{model}",
        model_settings=ModelSettings(
            extra_headers=get_extra_headers() or None,
            extra_args=get_timeout_extra_args(),
        ),
    )
    try:
        result = await Runner.run(agent, f"Question: {question}", max_turns=2)
    except MaxTurnsExceeded as exc:
        raise RuntimeError(
            f"Coverage grader hit the max-turn cap on question: {question!r}. "
            f"Try a more capable model."
        ) from exc
    raw = (result.final_output or "").strip()
    upper = raw.upper()
    verdict: Literal["supported", "unsupported", "ambiguous"]
    if "UNSUPPORTED" in upper:
        verdict = "unsupported"
    elif "SUPPORTED" in upper:
        verdict = "supported"
    else:
        # Grader didn't emit a parseable verdict — surface as a distinct
        # state so callers can report grader-malfunction separately from
        # "the body is hollow." See ``EvalResult.coverage_ambiguous``.
        verdict = "ambiguous"
    reason = ""
    for line in raw.splitlines():
        stripped = line.strip()