Function _run_harness

eval/run_compare.py:579–624

(
    *,
    swebench_repo: Path,
    swebench_python: str,
    dataset_name: str,
    split: str,
    predictions_path: Path,
    instance_ids: list[str] | None,
    run_id: str,
    max_workers: int,
    log_path: Path,
)

def _run_harness(
    *,
    swebench_repo: Path,
    swebench_python: str,
    dataset_name: str,
    split: str,
    predictions_path: Path,
    instance_ids: list[str] | None,
    run_id: str,
    max_workers: int,
    log_path: Path,
) -> None:
    cmd = [
        swebench_python,
        "-m",
        "swebench.harness.run_evaluation",
        "--dataset_name",
        dataset_name,
        "--split",
        split,
        "--predictions_path",
        str(predictions_path),
        "--max_workers",
        str(max_workers),
        "--run_id",
        run_id,
    ]
    if instance_ids:
        cmd.extend(["--instance_ids", *instance_ids])
    _info(f"  harness: {predictions_path.name} (run_id={run_id})")
    log_path.parent.mkdir(parents=True, exist_ok=True)
    try:
        with log_path.open("wb") as log_handle:
            subprocess.run(  # noqa: S603
                cmd,
                cwd=str(swebench_repo),
                check=True,
                stdout=log_handle,
                stderr=subprocess.STDOUT,
            )
    except subprocess.CalledProcessError as e:
        _info(
            f"harness subprocess failed (exit {e.returncode}). "
            f"Later agents were not started. Full log: {log_path}"
        )
        raise


def _find_summary(swebench_repo: Path, model_name: str, run_id: str) -> Path: