Function generate_eval_set

openkb/skill/evaluator.py:195–271

Use an LLM to generate ``count`` should-trigger + ``count`` should-not eval prompts grounded in what the skill actually claims to cover. The generator now sees the SKILL.md body and reference previews, not just the description. That keeps the should-trigger questions realistic (ques

(
    skill_dir: Path,
    *,
    model: str,
    count: int = EVAL_DEFAULT_COUNT,
)

async def generate_eval_set(
    skill_dir: Path,
    *,
    model: str,
    count: int = EVAL_DEFAULT_COUNT,
) -> list[EvalPrompt]:
    """Use an LLM to generate ``count`` should-trigger + ``count`` should-not
    eval prompts grounded in what the skill actually claims to cover.

    The generator now sees the SKILL.md body and reference previews, not
    just the description. That keeps the should-trigger questions
    realistic (questions the skill is genuinely set up to answer) and the
    should-not questions plausibly-adjacent (so trigger accuracy actually
    catches over-broad descriptions, not just disjoint ones).
    """
    desc = _read_description(skill_dir)
    content = _skill_content_block(skill_dir)

    instructions = (
        "You are designing an evaluation set for a knowledge-base skill. "
        "The skill's activation description is:\n\n"
        f"  {desc}\n\n"
        "The skill's body (SKILL.md without frontmatter) and references "
        "are below. Use them to ground the questions in what the skill "
        "actually claims to cover.\n\n"
        f"{content}\n\n"
        f"Produce exactly {count} 'should-trigger' user questions "
        "(questions a user might ask where this skill genuinely helps — "
        "ones the body and references contain material for) and exactly "
        f"{count} 'should-not' user questions (plausibly-adjacent "
        "questions where this skill is NOT the right tool — they should "
        "be close enough that a sloppy description would mis-trigger, "
        "but the body clearly does not cover them).\n\n"
        "Output ONLY a JSON object with this exact shape:\n"
        f'  {{"should_trigger": [...{count} strings...], '
        f'"should_not": [...{count} strings...]}}\n\n'
        "No prose. No markdown. Just the JSON object."
    )

    agent = Agent(
        name="eval-set-generator",
        instructions=instructions,
        model=f"litellm/{model}",
        model_settings=ModelSettings(
            extra_headers=get_extra_headers() or None,
            extra_args=get_timeout_extra_args(),
        ),
    )
    try:
        result = await Runner.run(agent, "Generate the eval set now.", max_turns=3)
    except MaxTurnsExceeded as exc:
        raise RuntimeError(
            "Eval set generation hit the max-turn cap. The model may be "
            "looping; try a different model or a smaller --count."
        ) from exc
    raw = (result.final_output or "").strip()

    # Strip optional code fence