Function save_eval_set

openkb/skill/evaluator.py:465–479

Persist an eval set to `` /.openkb/eval-sets/ .json``.

(
    kb_dir: Path,
    skill_name: str,
    prompts: list[EvalPrompt],
)

def save_eval_set(
    kb_dir: Path,
    skill_name: str,
    prompts: list[EvalPrompt],
) -> Path:
    """Persist an eval set to ``<kb>/.openkb/eval-sets/<skill_name>.json``."""
    out_dir = kb_dir / ".openkb" / "eval-sets"
    out_dir.mkdir(parents=True, exist_ok=True)
    out_path = out_dir / f"{skill_name}.json"
    data = {
        "should_trigger": [p.question for p in prompts if p.expected == "trigger"],
        "should_not": [p.question for p in prompts if p.expected == "no-trigger"],
    }
    out_path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
    return out_path


def load_eval_set(path: Path) -> list[EvalPrompt]: