Function generateEvalTask

evals/buffbench/eval-task-generator.ts:96–163

({
  client,
  input,
  agentDefinitions,
}: {
  client: CodebuffClient
  input: {
    commitSha: string
    parentSha: string
    diff: string
    editedFilePaths: string[]
    commitMessage?: string
    repoPath: string
  }
  agentDefinitions?: any[]
})

}

export async function generateEvalTask({
  client,
  input,
  agentDefinitions,
}: {
  client: CodebuffClient
  input: {
    commitSha: string
    parentSha: string
    diff: string
    editedFilePaths: string[]
    commitMessage?: string
    repoPath: string
  }
  agentDefinitions?: any[]
}): Promise<{
  id: string
  reasoning: string
  spec: string
  prompt: string
  supplementalFiles: string[]
}> {
  const { diff, editedFilePaths, commitMessage, repoPath } = input

  const allAgentDefinitions = [
    evalTaskGeneratorAgentDef,
    fileExplorerDef,
    findAllReferencerDef,
    ...(agentDefinitions || []),
  ]

  const generatorResult = await client.run({
    agent: 'eval-task-generator',
    prompt:
      'Generate a task specification and user prompt based on the git diff and codebase exploration',
    params: {
      diff,
      editedFilePaths,
      commitMessage,
    },
    cwd: repoPath,
    agentDefinitions: allAgentDefinitions,
    handleEvent: (event) => {
      if (event.type === 'subagent_start') {
        console.log(`[Agent] Starting: ${event.displayName}`)
      } else if (event.type === 'tool_call') {
        console.log(`[Tool] ${event.toolName}`)
      } else if (event.type === 'text') {
        console.log(`[Text] ${event.text}...`)
      }
    },
  })

  if (
    generatorResult.output.type !== 'structuredOutput' ||
    !generatorResult.output.value
  ) {
    throw new Error('Failed to generate structured task output')