(argv)
| 623 | |
| 624 | |
| 625 | def main(argv): |
| 626 | args = parse_args(argv) |
| 627 | args.tracedecay_bin = resolve_tracedecay_bin(args.tracedecay_bin) |
| 628 | scenarios = load_scenarios(args.scenario) |
| 629 | if not scenarios: |
| 630 | sys.exit("no runnable scenarios selected") |
| 631 | |
| 632 | timestamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") |
| 633 | run_dir = RUNS_DIR / timestamp |
| 634 | run_dir.mkdir(parents=True, exist_ok=True) |
| 635 | report = { |
| 636 | "schema_version": 1, |
| 637 | "timestamp": timestamp, |
| 638 | "driver": args.driver, |
| 639 | "model": args.model, |
| 640 | "tracedecay_bin": str(args.tracedecay_bin), |
| 641 | "scenarios": [], |
| 642 | } |
| 643 | |
| 644 | if not (args.agent_turn and args.i_understand_model_cost): |
| 645 | report["status"] = "blocked" |
| 646 | report["reason"] = ( |
| 647 | "real-model turns are cost-gated: pass both --agent-turn and " |
| 648 | "--i-understand-model-cost to run them" |
| 649 | ) |
| 650 | report["requested_scenarios"] = [s["id"] for s in scenarios] |
| 651 | report_path = run_dir / "report.json" |
| 652 | report_path.write_text(json.dumps(report, indent=2) + "\n") |
| 653 | print(json.dumps(report, indent=2)) |
| 654 | print(f"\nblocked report written to {report_path}", file=sys.stderr) |
| 655 | return 2 |
| 656 | |
| 657 | overall_ok = True |
| 658 | for scenario in scenarios: |
| 659 | eval_env = create_eval_environment(scenario["id"]) |
| 660 | args.hermes_profile = resolve_hermes_profile(args) |
| 661 | fixture = None |
| 662 | try: |
| 663 | fixture, db_path = build_fixture(scenario, args.tracedecay_bin, eval_env.env) |
| 664 | if args.driver == "hermes": |
| 665 | transcripts = drive_hermes(args, scenario, fixture, run_dir, eval_env.env) |
| 666 | else: |
| 667 | transcripts = drive_cursor_agent(args, scenario, fixture, run_dir, eval_env.env) |
| 668 | outcomes = evaluate_assertions(scenario, db_path) |
| 669 | failed = [o for o in outcomes if not o["passed"]] |
| 670 | status = "pass" if not failed else "fail" |
| 671 | if failed and scenario.get("contract") == "pending-sibling": |
| 672 | status = "fail (note: scenario contract is pending-sibling — see contract_notes)" |
| 673 | if not all(t["turn_valid"] for t in transcripts): |
| 674 | status = "error (agent turn invalid — see transcript logs)" |
| 675 | failed = failed or [{"name": "agent-turn", "passed": False}] |
| 676 | report["scenarios"].append( |
| 677 | { |
| 678 | "id": scenario["id"], |
| 679 | "contract": scenario.get("contract", "stable"), |
| 680 | "status": status, |
| 681 | "assertions": outcomes, |
| 682 | "transcripts": transcripts, |
no test coverage detected