Function runTask

evals/buffbench/run-buffbench.ts:44–263

(options: {
  client: CodebuffClient
  commit: EvalDataV2['evalCommits'][0]
  agents: string[]
  repoUrl: string
  initCommand?: string
  env?: Record<string, string>
  logsDir: string
  index: number
  totalTasks: number
  analyzerContext: {
    agentDefinitions: any[]
    agentTypeDefinition: string
    testedAgentIds: string[]
  }
  localAgentDefinitions: any[]
  extractLessons: boolean
  printEvents: boolean
  finalCheckCommands?: string[]
  disableAnalysis?: boolean
  saveTraces?: boolean
})

}

async function runTask(options: {
  client: CodebuffClient
  commit: EvalDataV2['evalCommits'][0]
  agents: string[]
  repoUrl: string
  initCommand?: string
  env?: Record<string, string>
  logsDir: string
  index: number
  totalTasks: number
  analyzerContext: {
    agentDefinitions: any[]
    agentTypeDefinition: string
    testedAgentIds: string[]
  }
  localAgentDefinitions: any[]
  extractLessons: boolean
  printEvents: boolean
  finalCheckCommands?: string[]
  disableAnalysis?: boolean
  saveTraces?: boolean
}) {
  const {
    client,
    commit,
    agents,
    repoUrl,
    initCommand,
    env,
    logsDir,
    index,
    totalTasks,
    analyzerContext,
    localAgentDefinitions,
    extractLessons,
    printEvents,
    finalCheckCommands,
    disableAnalysis,
    saveTraces = false,
  } = options

  console.log(
    `\n=== Task ${index + 1}/${totalTasks}: ${commit.id} (${commit.sha.slice(0, 7)}) ===`,
  )

  // Store trace data for this commit to analyze later
  const commitTraces: AgentTraceData[] = []

  const agentPromises = agents.map(async (agent) => {
    const { agentId, externalAgentType } = parseAgentId(agent)

    const agentResult = await runAgentOnCommit({
      client,
      agentId,
      commit,
      repoUrl,
      initCommand,
      env,