Function extractAgentLessons

evals/buffbench/lessons-extractor.ts:92–187

(
  input: ExtractAgentLessonsInput,
)

}

export async function extractAgentLessons(
  input: ExtractAgentLessonsInput,
): Promise<{
  lessons: Lesson[]
}> {
  try {
    const {
      client,
      localAgentDefinitions,
      prompt,
      groundTruthFileDiffs,
      contextFiles,
      agentDiff,
      agentTrace,
      judgeResult,
      error,
    } = input

    const groundTruthDiffs = groundTruthFileDiffs
      .map(({ path, diff }) => `### ${path}\n\`\`\`diff\n${diff}\n\`\`\``)
      .join('\n\n')

    const contextFilesContent = Object.entries(contextFiles)
      .map(
        ([filePath, content]) => `### ${filePath}\n\`\`\`\n${content}\n\`\`\``,
      )
      .join('\n\n')

    const truncated = truncateTrace(agentTrace)
    const truncatedTraceJson = JSON.stringify(truncated, null, 2)

    const judgeSummary = judgeResult
      ? `Overall: ${judgeResult.overallScore}/10, Completion: ${judgeResult.completionScore}/10, Code Quality: ${judgeResult.codeQualityScore}/10
Weaknesses: ${judgeResult.weaknesses?.join(' | ') || '(none)'}`
      : '(no judge summary)'

    const lessonsPrompt = `## User Prompt
${prompt}

## Context Files (from parent commit)
${contextFilesContent || '(No context files)'}

## Ground Truth Changes (One valid implementation)
${groundTruthDiffs || '(No ground truth diffs)'}

## Agent's Changes
\`\`\`diff
${agentDiff || '(No changes made)'}
\`\`\`

## Agent Trace (truncated)
\`\`\`json
${truncatedTraceJson}
\`\`\`

## Judge Summary
${judgeSummary}
${error ? `\n## Agent Error\n${error}\n` : ''}