Function evalPlannerAgent

evals/subagents/eval-planner.ts:16–158

(params: {
  client: CodebuffClient
  agentId: string
  agentDefinitions: Array<AgentDefinition>
  spec: string
  repoUrl: string
  parentSha: string
  initCommand?: string
  fileStates: Array<{
    path: string
    preContent: string
    postContent: string
  }>
})

import type { AgentDefinition } from '@codebuff/sdk'

export const evalPlannerAgent = async (params: {
  client: CodebuffClient
  agentId: string
  agentDefinitions: Array<AgentDefinition>
  spec: string
  repoUrl: string
  parentSha: string
  initCommand?: string
  fileStates: Array<{
    path: string
    preContent: string
    postContent: string
  }>
}) => {
  const {
    client,
    agentId,
    agentDefinitions,
    spec,
    repoUrl,
    parentSha,
    initCommand,
    fileStates,
  } = params
  const plannerStartTime = Date.now()
  const result = await withTestRepo(
    { repoUrl, parentSha, initCommand },
    async (cwd) => {
      // Run the agent with the test repository as cwd
      console.log(`Running agent ${agentId} with prompt: ${spec}...`)
      return await client.run({
        agent: agentId,
        prompt: `Please plan a full implementation of the following spec: ${spec}`,
        cwd,
        agentDefinitions,
        handleEvent: (event) => {
          console.log(agentId, JSON.stringify(event, null, 2))
        },
      })
    },
  )
  const plannerLatencyMs = Date.now() - plannerStartTime

  const { output } = result

  const outputString = JSON.stringify(
    'value' in output ? output.value : output.message,
  )

  // Compute file changes and diffs
  const fileChangesSection = fileStates
    .map(({ path, preContent, postContent }) => {
      return `\n### File: ${path}\n\n<pre_content>\n${preContent}\n</pre_content>\n\n<post_content>\n${postContent}\n</post_content>`
    })
    .join('\n')

  const diffsSection = fileStates
    .map(({ path, preContent, postContent }) => {