Function grade_one

openkb/skill/evaluator.py:274–318

Ask the trigger grader LLM whether the description suggests this skill should be loaded for the given question. The grader deliberately sees ONLY the description — that is the trigger surface external agents will see when deciding to load the skill. The body is irrelevant to *this*

(
    description: str,
    question: str,
    *,
    model: str,
)

async def grade_one(
    description: str,
    question: str,
    *,
    model: str,
) -> Literal["trigger", "no-trigger"]:
    """Ask the trigger grader LLM whether the description suggests this
    skill should be loaded for the given question.

    The grader deliberately sees ONLY the description — that is the
    trigger surface external agents will see when deciding to load the
    skill. The body is irrelevant to *this* metric; see
    :func:`grade_coverage` for the body-aware check.
    """
    instructions = (
        "You are deciding whether an agent should load a specific skill to "
        "answer a user question. You will be given the skill's activation "
        "description and a single user question. Answer with one word: "
        "TRIGGER (load the skill) or NO-TRIGGER (don't load).\n\n"
        f"Skill description:\n  {description}\n\n"
        "Reply with exactly one of: TRIGGER, NO-TRIGGER."
    )
    agent = Agent(
        name="trigger-grader",
        instructions=instructions,
        model=f"litellm/{model}",
        model_settings=ModelSettings(
            extra_headers=get_extra_headers() or None,
            extra_args=get_timeout_extra_args(),
        ),
    )
    try:
        result = await Runner.run(agent, f"Question: {question}", max_turns=2)
    except MaxTurnsExceeded as exc:
        raise RuntimeError(
            f"Trigger grader hit the max-turn cap on question: {question!r}. "
            f"Try a more capable model."
        ) from exc
    raw = (result.final_output or "").strip().upper()
    if "NO-TRIGGER" in raw or "NO TRIGGER" in raw:
        return "no-trigger"
    if "TRIGGER" in raw:
        return "trigger"
    # Default: assume no-trigger on ambiguous output
    return "no-trigger"


async def grade_coverage(