Function run_instance

src/benchmarks/elaipbench/instance.py:22–117

Run the agent on a single ELAIPBench question.

(
    idx: int,
    total: int,
    instance: dict,
    args,
    output_dir: Path,
)

def run_instance(
    idx: int,
    total: int,
    instance: dict,
    args,
    output_dir: Path,
) -> ELAIPBenchResult:
    """Run the agent on a single ELAIPBench question."""
    qid = instance["id"]
    qtype = instance["question_type"]
    preview = instance["question"][:80].replace("\n", " ")

    _safe_print(f"\n[{idx}/{total}] Question {qid} ({qtype}): {preview}...")

    try:
        from harness.agent import AgentSPEX
        from harness.types.config import EffectiveArgs
        from mcp_client.client import MCPClient

        mcp_client = MCPClient()
        agent = AgentSPEX(mcp_client=mcp_client)

        custom_yaml_file = prepare_yaml_for_question(
            args.workflow_file,
            instance,
            output_dir,
            qtype,
        )

        # Create per-question log directory
        question_log_dir = output_dir / f"logs/question_{qid}"
        question_log_dir.mkdir(parents=True, exist_ok=True)

        # Create a copy of args with per-question output_dir for isolated logging
        class _ArgsWithOutputDir:
            def __init__(self, original_args, output_dir):
                self.__dict__.update(
                    {
                        k: getattr(original_args, k)
                        for k in dir(original_args)
                        if not k.startswith("__")
                    }
                )
                self.output_dir = str(output_dir)

        per_question_args = _ArgsWithOutputDir(args, question_log_dir)

        agent_args = EffectiveArgs(
            workflow_file=custom_yaml_file,
            model=args.model if args.model else "gpt-4.1",
            _original_args=per_question_args,
        )

        agent_output = agent.run(agent_args)
        response_text = (
            agent_output if isinstance(agent_output, str) else str(agent_output or "")
        )