Function main

eval/hermetic/score.py:297–346

()

def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--agent", choices=("claude", "codex"), default="claude")
    ap.add_argument("--scenario", required=True, help="scenario JSON (one line)")
    ap.add_argument("--claude-json", help="path to claude -p json result")
    ap.add_argument("--codex-jsonl", help="path to codex exec --json output")
    ap.add_argument("--config-dir", help="isolated CLAUDE_CONFIG_DIR")
    ap.add_argument("--cwd", required=True, help="cwd the scenario ran in")
    ap.add_argument(
        "--verify-status",
        type=int,
        choices=(0, 1),
        default=None,
        help="exit status from the scenario verify_cmd (0=pass, 1=fail)",
    )
    ap.add_argument("--rep", type=int, default=1, help="corpus repetition index")
    args = ap.parse_args()

    scenario = load_scenario(args.scenario)
    sid = None
    transcript = None
    td_tools: list[str] = []
    native_tools: list[str] = []
    commands: list[str] = []

    if args.agent == "claude":
        if not args.claude_json or not args.config_dir:
            ap.error("--agent claude requires --claude-json and --config-dir")
        sid = session_id_from_claude_json(Path(args.claude_json))
        transcript = find_transcript(Path(args.config_dir), args.cwd, sid)
        if transcript is not None:
            td_tools, native_tools, commands = count_claude_tools(transcript)
    else:
        if not args.codex_jsonl:
            ap.error("--agent codex requires --codex-jsonl")
        transcript = Path(args.codex_jsonl)
        td_tools, native_tools, commands = count_codex_tools(transcript)

    result = evaluate_scenario(
        scenario,
        sid,
        transcript,
        td_tools,
        native_tools,
        commands,
        verify_status=args.verify_status,
        rep=args.rep,
    )
    print(json.dumps(result))
    return 0


if __name__ == "__main__":