Function judgeCommitResult

evals/buffbench/judge.ts:214–303

(
  input: JudgeCommitResultInput,
)

}

export async function judgeCommitResult(
  input: JudgeCommitResultInput,
): Promise<JudgingResult> {
  const { commit, contextFiles, agentDiff, error, finalCheckOutputs } = input

  const { prompt, fileDiffs } = commit

  const groundTruthDiffs = fileDiffs
    .map(({ path, diff }) => {
      return `### ${path}\n\`\`\`diff\n${diff}\n\`\`\``
    })
    .join('\n\n')

  const contextFilesContent = Object.entries(contextFiles)
    .map(([filePath, content]) => {
      return `### ${filePath}\n\`\`\`\n${content}\n\`\`\``
    })
    .join('\n\n')

  const judgePrompt = `## User Prompt (What the agent was asked to do)
${prompt}

## Context Files (from parent commit)
${contextFilesContent || '(No context files)'}

## Ground Truth Changes (One valid implementation)
${groundTruthDiffs}

## Agent's Changes (What the agent actually did)
\`\`\`diff
${agentDiff || '(No changes made)'}
\`\`\`
${error ? `\n## Error Encountered\n${error}` : ''}
${finalCheckOutputs ? `\n## Final Check Command Outputs\n${finalCheckOutputs}` : ''}`

  // Run 2 judges in parallel
  const judgePromises = [
    runSingleJudge(input, judgePrompt, 'judge-gpt'),
    runSingleJudge(input, judgePrompt, 'judge-gemini'),
  ]

  const judgeResults = await Promise.all(judgePromises)
  const validResults = judgeResults.filter(
    (result): result is JudgingResult => result !== null,
  )

  if (validResults.length === 0) {
    console.error('All judges failed to provide results')
    return {
      analysis: 'Error running judge agent - all judges failed',
      strengths: [],
      weaknesses: ['All judges failed to provide structured output'],
      completionScore: 0,
      codeQualityScore: 0,
      overallScore: 0,
    }
  }