Function runBuffBench

evals/buffbench/run-buffbench.ts:319–656

(options: {
  evalDataPaths: string[]
  agents: string[]
  taskConcurrency?: number
  client?: CodebuffClient
  taskIds?: string[]
  extractLessons?: boolean
  disableAnalysis?: boolean
  saveTraces?: boolean
})

}

export async function runBuffBench(options: {
  evalDataPaths: string[]
  agents: string[]
  taskConcurrency?: number
  client?: CodebuffClient
  taskIds?: string[]
  extractLessons?: boolean
  disableAnalysis?: boolean
  saveTraces?: boolean
}) {
  const {
    evalDataPaths,
    agents,
    taskConcurrency = 1,
    taskIds,
    extractLessons = false,
    disableAnalysis = false,
    saveTraces = false,
  } = options

  if (evalDataPaths.length === 0) {
    throw new Error('At least one eval data path is required')
  }

  // Load all eval files and create a mapping of commits to their source eval data
  const allCommitsWithSource: CommitWithSource[] = []
  const loadedEvalFiles: { path: string; data: EvalDataV2 }[] = []

  for (const evalDataPath of evalDataPaths) {
    const evalData: EvalDataV2 = JSON.parse(
      fs.readFileSync(evalDataPath, 'utf-8'),
    )
    loadedEvalFiles.push({ path: evalDataPath, data: evalData })

    for (const commit of evalData.evalCommits) {
      allCommitsWithSource.push({
        commit,
        evalData,
        evalDataPath,
      })
    }
  }

  console.log(
    `Loaded ${loadedEvalFiles.length} eval file(s) with ${allCommitsWithSource.length} total tasks`,
  )
  for (const { path: p, data } of loadedEvalFiles) {
    console.log(`  - ${path.basename(p)}: ${data.evalCommits.length} tasks`)
  }

  // Collect all unique binInstalls from all eval files
  const allBinInstalls = loadedEvalFiles.flatMap(
    (f) => f.data.binInstalls ?? [],
  )
  const uniqueBinInstalls = allBinInstalls.filter(
    (bin, index, self) => index === self.findIndex((b) => b.name === bin.name),
  )