Function main

evals/subagents/eval-planner.ts:223–396

()

}

async function main() {
  // Load the eval file
  const evalFilePath = path.join(
    __dirname,
    '..',
    'git-evals',
    'eval-codebuff2.json',
  )
  const evalData: EvalData = JSON.parse(fs.readFileSync(evalFilePath, 'utf-8'))

  const { repoUrl, initCommand, evalCommits } = evalData

  const client = new CodebuffClient({
    apiKey: process.env[API_KEY_ENV_VAR] || getUserCredentials()?.authToken,
  })

  const agentsPath = path.join(__dirname, '../../.agents')
  const localAgentDefinitions = Object.values(
    await loadLocalAgents({
      agentsPath,
    }),
  )

  const allResults = [] as Array<{
    sha: string
    spec: string
    agentOutput: string
    judgingResults: {
      reasoning: string
      pros: string
      cons: string
      overallScore: number
    }
    plannerLatencyMs: number
  }>

  // Track statistics
  const stats = {
    total: evalCommits.length,
    completed: 0,
    failed: 0,
    scores: [] as number[],
    plannerLatencies: [] as number[],
  }

  // Loop through each eval task
  for (const evalCommit of evalCommits) {
    const { sha, parentSha, spec, fileStates } = evalCommit

    console.log(`\n=== Running eval for commit ${sha} ===`)
    console.log(`Spec: ${spec.substring(0, 100)}...\n`)

    try {
      const result = await evalPlannerAgent({
        client,
        agentId: 'implementation-planner',
        agentDefinitions: localAgentDefinitions,
        spec,