Function run_instance

src/benchmarks/swe_bench_verified/instance.py:152–276

Run the agent on a single SWE-Bench instance.

(
    instance: Dict[str, Any],
    agent: "AgentSPEX",
    mcp_client: "MCPClient",
    args,
    output_dir: Path,
    save_logs: bool = False,
)

def run_instance(
    instance: Dict[str, Any],
    agent: "AgentSPEX",
    mcp_client: "MCPClient",
    args,
    output_dir: Path,
    save_logs: bool = False,
) -> InstanceResult:
    """Run the agent on a single SWE-Bench instance."""
    instance_id = instance["instance_id"]
    problem_statement = instance["problem_statement"]

    host_log_path = output_dir / f"{instance_id}_full.log" if save_logs else None
    log_file = str(output_dir / f"{instance_id}_full.log")
    event_log_file = str(output_dir / f"{instance_id}_agent_events.log")
    host_event_log_path = output_dir / f"{instance_id}_agent_events.log"
    logger = Logger(
        mcp_client=mcp_client,
        log_file=log_file,
        host_log_path=host_log_path,
        event_log_path=event_log_file,
        host_event_log_path=host_event_log_path,
        deduplicate=True,
    )

    logger(f"\n{'#'*80}")
    logger(f"Processing instance: {instance_id}")
    logger(f"{'#'*80}\n")

    try:
        repo_path = "/testbed"
        conda_env_to_use = "testbed"

        custom_yaml_file = prepare_yaml_for_instance(
            args.workflow_file,
            instance,
            repo_path,
            output_dir,
            conda_env_name=conda_env_to_use,
        )

        # Get model from CLI args, or fall back to workflow config
        model = getattr(args, "model", None)
        if not model:
            # Read model from workflow config
            with open(args.workflow_file, "r") as f:
                workflow = yaml.safe_load(f)
            model = workflow.get("config", {}).get("model", "gpt-4.1-mini")

        agent_args = AgentArgs(
            workflow_file=custom_yaml_file,
            model=model,
            problem_statement=problem_statement,
        )

        logger(f"Running agent on {instance_id}...")

        _ = agent.run(agent_args, external_logger=logger)