Function skill_eval

openkb/cli.py:2632–2744

Measure how accurately a compiled skill's description fires. Generates trigger-eval prompts via LLM, then asks a grader LLM whether the description should activate the skill for each prompt. Prints pass rate + miss list.

(ctx, name, save_flag, eval_set_path, count)

)
@click.pass_context
def skill_eval(ctx, name, save_flag, eval_set_path, count):
    """Measure how accurately a compiled skill's description fires.

    Generates trigger-eval prompts via LLM, then asks a grader LLM whether
    the description should activate the skill for each prompt. Prints pass
    rate + miss list.
    """
    from openkb.skill.evaluator import (
        run_eval,
        save_eval_set,
        load_eval_set,
        EvalPrompt,
    )

    from openkb.skill import skill_dir as _skill_dir

    kb_dir = _find_kb_dir(ctx.obj.get("kb_dir_override"))
    if kb_dir is None:
        click.echo("No knowledge base found. Run `openkb init` first.", err=True)
        ctx.exit(1)

    skill_dir = _skill_dir(kb_dir, name)
    if not skill_dir.is_dir():
        click.echo(f"[ERROR] Skill '{name}' not found.", err=True)
        ctx.exit(1)

    try:
        _setup_llm_key(kb_dir)
    except RuntimeError as exc:
        click.echo(f"[ERROR] {exc}", err=True)
        ctx.exit(1)
    config = load_config(kb_dir / ".openkb" / "config.yaml")
    model = config.get("model", DEFAULT_CONFIG["model"])

    eval_set: list[EvalPrompt] | None = None
    if eval_set_path:
        eval_set = load_eval_set(Path(eval_set_path))
        click.echo(f"Loaded eval set from {eval_set_path} ({len(eval_set)} prompts).")
    else:
        click.echo(f"Generating eval set for '{name}' (count={count} per side)...")

    try:
        result = asyncio.run(
            run_eval(
                skill_dir,
                model=model,
                eval_set=eval_set,
                count=count,
            )
        )
    except RuntimeError as exc:
        click.echo(f"[ERROR] {exc}", err=True)
        ctx.exit(1)

    click.echo(f"\nEval set: {result.total} prompts")
    click.echo(
        f"Trigger accuracy: {result.passed}/{result.trigger_scored} "
        f"({result.pass_rate * 100:.0f}%)  "