Function analyzeAgentTraces

evals/buffbench/trace-analyzer.ts:110–230

({
  client,
  traces,
  codingAgentPrompt,
  analyzerContext,
}: {
  client: CodebuffClient
  traces: AgentTraceData[]
  codingAgentPrompt: string
  analyzerContext: {
    agentDefinitions: any[]
    agentTypeDefinition: string
    testedAgentIds: string[]
  }
})

}

export async function analyzeAgentTraces({
  client,
  traces,
  codingAgentPrompt,
  analyzerContext,
}: {
  client: CodebuffClient
  traces: AgentTraceData[]
  codingAgentPrompt: string
  analyzerContext: {
    agentDefinitions: any[]
    agentTypeDefinition: string
    testedAgentIds: string[]
  }
}): Promise<{
  overallAnalysis: string
  agentFeedback: Array<{
    agentId: string
    strengths: string[]
    weaknesses: string[]
    recommendations: string[]
  }>
}> {
  try {
    const truncatedTraces = traces.map((t) => ({
      agentId: t.agentId,
      trace: truncateTrace(t.trace),
      judgeResult: t.judgeResult,
      cost: t.cost,
      durationMs: t.durationMs,
      error: t.error,
    }))

    // Filter agent definitions to only include tested agents
    const filteredAgentDefinitions = analyzerContext.agentDefinitions.filter(
      (def) => analyzerContext.testedAgentIds.includes(def.id),
    )

    const prompt = `## Agent Definitions Being Evaluated

Below are the complete agent definitions for the agents being tested. Use this to understand their configuration, tools, prompts, and capabilities.

${JSON.stringify(filteredAgentDefinitions, null, 2)}

## Agent Type Definition Reference

For reference, here is the TypeScript type definition that agents use:

\`\`\`typescript
${analyzerContext.agentTypeDefinition}
\`\`\`

## Coding Agent Prompt (for context)
${codingAgentPrompt}

## Agent Traces and Results
${JSON.stringify(truncatedTraces, null, 2)}