Function analyzeAllTasks

evals/buffbench/meta-analyzer.ts:146–287

(params: {
  client: CodebuffClient
  logsDir: string
  agents: string[]
  analyzerContext: {
    agentDefinitions: any[]
    agentTypeDefinition: string
    testedAgentIds: string[]
  }
})

}

export async function analyzeAllTasks(params: {
  client: CodebuffClient
  logsDir: string
  agents: string[]
  analyzerContext: {
    agentDefinitions: any[]
    agentTypeDefinition: string
    testedAgentIds: string[]
  }
}): Promise<MetaAnalysisResult> {
  const { client, logsDir, agents, analyzerContext } = params

  try {
    // Read all ANALYSIS files from logs directory
    const files = fs.readdirSync(logsDir)
    const analysisFiles = files.filter((f) => f.includes('ANALYSIS'))

    const allTaskAnalyses: TaskAnalysisData[] = []
    for (const file of analysisFiles) {
      const filePath = path.join(logsDir, file)
      const content = fs.readFileSync(filePath, 'utf-8')
      const data: TaskAnalysisData = JSON.parse(content)
      allTaskAnalyses.push(data)
    }

    if (allTaskAnalyses.length === 0) {
      console.warn('No analysis files found in logs directory')
      return {
        overallComparison: 'No analysis data available',
        agentInsights: [],
        keyFindings: [],
      }
    }

    // Create a concise summary for each task (without full agent traces)
    const taskSummaries = allTaskAnalyses.map((task) => ({
      prompt: task.prompt,
      traceAnalysis: {
        overallAnalysis: task.overallAnalysis,
        agentFeedback: task.agentFeedback,
      },
      judgeResults: task.results.map((r) => ({
        agentId: r.agentId,
        overallScore: r.overallScore,
        completionScore: r.completionScore,
        codeQualityScore: r.codeQualityScore,
        cost: r.cost,
        durationMs: r.durationMs,
        strengths: r.strengths,
        weaknesses: r.weaknesses,
        error: r.error,
      })),
    }))

    // Filter agent definitions to only include tested agents
    const filteredAgentDefinitions = analyzerContext.agentDefinitions.filter(
      (def) => analyzerContext.testedAgentIds.includes(def.id),
    )