Function main

evals/buffbench/main-hard-tasks.ts:15–42

()

}

async function main() {
  const saveTraces = process.argv.includes('--save-traces')

  const evalPaths = [
    path.join(__dirname, 'eval-codebuff2.json'),
    path.join(__dirname, 'eval-manifold2.json'),
    path.join(__dirname, 'eval-plane2.json'),
    path.join(__dirname, 'eval-saleor2.json'),
  ]

  // Load all task IDs from the eval files
  const allTaskIds = evalPaths.flatMap(loadTaskIds)

  console.log(
    `Running ${allTaskIds.length} hard tasks across ${evalPaths.length} eval sets`,
  )

  // Run all hard tasks across all 4 eval sets
  await runBuffBench({
    evalDataPaths: evalPaths,
    agents: ['base2', 'external:claude'],
    taskIds: allTaskIds,
    taskConcurrency: 4,
    saveTraces,
  })

  process.exit(0)
}

if (import.meta.main) {
  main().catch((error) => {