Function _run_single_problem

src/benchmarks/aime/run.py:105–190

Run the agent on a single AIME problem.

(
    idx: int,
    total: int,
    problem: dict,
    args,
    output_dir: Path,
)

def _run_single_problem(
    idx: int,
    total: int,
    problem: dict,
    args,
    output_dir: Path,
) -> AIMEResult:
    """Run the agent on a single AIME problem."""
    pid = problem["id"]
    preview = problem["problem"][:80].replace("\n", " ")

    print(f"\n[{idx}/{total}] Problem {pid} ({problem['dataset']}): {preview}...")

    try:
        from harness.agent import AgentSPEX
        from harness.types.config import EffectiveArgs
        from mcp_client.client import MCPClient

        mcp_client = MCPClient()
        agent = AgentSPEX(mcp_client=mcp_client)

        # Create a thread-local copy of args to avoid race conditions
        # Ensure agent-level resume is disabled (benchmark handles its own resume)
        local_args = argparse.Namespace(**vars(args))
        local_args.problem_statement = QUESTION_PROMPT.format(
            problem=problem["problem"]
        )
        local_args.resume = False

        agent_args = EffectiveArgs(
            workflow_file=args.workflow_file,
            _original_args=local_args,
        )

        import concurrent.futures

        executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
        future = executor.submit(agent.run, agent_args)
        timeout = getattr(args, "timeout", 900)
        try:
            agent_output = future.result(timeout=timeout)
        finally:
            executor.shutdown(wait=False, cancel_futures=True)
        response_text = (
            agent_output if isinstance(agent_output, str) else str(agent_output or "")
        )

        parsed = parse_answer(response_text)
        ground_truth = problem["answer"]

        # Use math-verify for robust comparison
        is_correct = verify_answer(response_text, ground_truth)

        result = AIMEResult(
            problem_id=pid,
            problem=problem["problem"],
            correct_answer=ground_truth,
            dataset=problem["dataset"],