Function runSingleJudge

evals/buffbench/judge.ts:150–212

(
  input: JudgeCommitResultInput,
  judgePrompt: string,
  judgeAgentId: string,
)

}

async function runSingleJudge(
  input: JudgeCommitResultInput,
  judgePrompt: string,
  judgeAgentId: string,
): Promise<JudgingResult | null> {
  const { client } = input

  const judgeAgent = judgeAgents[judgeAgentId]
  const agentOutput: string[] = []
  try {
    const judgeResult = await withTimeout(
      client.run({
        agent: judgeAgent.id,
        prompt: judgePrompt,
        agentDefinitions: Object.values(judgeAgents),
        handleEvent: (event) => {
          if (event.type === 'text') {
            agentOutput.push(event.text)
          } else if (event.type === 'tool_call') {
            agentOutput.push(JSON.stringify(event, null, 2))
          } else if (event.type === 'error') {
            console.warn(`[Judge ${judgeAgentId}] Error event:`, event.message)
          }
        },
      }),
      20 * 60 * 1000,
      'Judge agent timed out after 20 minutes',
    )

    if (judgeResult.output.type !== 'structuredOutput') {
      console.error(
        `Judge ${judgeAgentId} - not structured output`,
        JSON.stringify(judgeResult.output, null, 2),
      )
      console.error(
        'Judge agent output:',
        JSON.stringify(judgeResult.output, null, 2),
        'Judge agent output trace:',
        agentOutput.join(''),
      )
      if (DEBUG_ERROR) {
        fs.writeFileSync(
          path.join(
            __dirname,
            '..',
            `${input.commit.id}-${judgeAgentId}-agent-output-error.json`,
          ),
          JSON.stringify(
            { output: judgeResult.output, trace: agentOutput },
            null,
            2,
          ),
        )
      }
      return null
    }

    return judgeResult.output.value as JudgingResult