Function cmd_run

eval/run_compare.py:648–772

(args: argparse.Namespace)

def cmd_run(args: argparse.Namespace) -> int:
    swebench_repo = _resolve_swebench_repo(args.swebench_repo)
    swebench_python = _resolve_python("SWEBENCH_PYTHON")
    openclaude_repo = _resolve_openclaude_repo(args.openclaude_repo)
    clawcodex_repo = _resolve_clawcodex_repo(args.clawcodex_repo)

    dataset_local = swebench_repo / args.dataset_local
    if not dataset_local.is_dir():
        py_hint = _resolve_python("SWEBENCH_PYTHON")
        raise SystemExit(
            "Text dataset not found.\n\n"
            f"  Expected directory: {dataset_local}\n\n"
            "Build it once with:\n\n"
            f"  {py_hint} eval/run_compare.py prepare \\\n"
            f"    --swebench-repo {swebench_repo}\n\n"
            "That Python must have the `swebench` package (editable install from your "
            "SWE-bench-dev checkout). For example:\n\n"
            f"  {py_hint} -m pip install -e {swebench_repo} fastapi uvicorn tiktoken transformers\n\n"
            "Then re-run `run`. If the dataset lives elsewhere, pass "
            "`--dataset-local <relative-path-under-SWE-bench-dev>`."
        )

    instance_ids: list[str] | None = None
    if args.scope == "smoke":
        instance_ids = list(args.smoke_instances or DEFAULT_SMOKE_INSTANCES)
    elif args.scope == "instances":
        instance_ids = [i.strip() for i in args.instance_ids.split(",") if i.strip()]
        if not instance_ids:
            raise SystemExit("--scope=instances requires --instance-ids")

    timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
    run_id_base = args.run_id or f"compare-{timestamp}"
    run_dir = (EVAL_DIR / "runs" / run_id_base).resolve()
    run_dir.mkdir(parents=True, exist_ok=True)
    paths = RunPaths(run_dir=run_dir, run_id=run_id_base)

    specs = _build_agent_specs(args)

    # Sequential mode: predictions → harness for each agent in turn.
    summary_paths: dict[str, Path] = {}
    for spec in specs:
        _info(f"== {spec.name} ==")
        env = _server_env(
            swebench_repo=swebench_repo,
            clawcodex_repo=clawcodex_repo,
            openclaude_repo=openclaude_repo,
            agent=spec.name,
        )
        env.update(spec.env_overrides)

        with _spawn_server(
            swebench_repo=swebench_repo,
            swebench_python=swebench_python,
            server_module=spec.server_module,
            port=spec.port,
            log_path=paths.server_logs[spec.name],
            env=env,
        ):