Function generateEvalFileV2

evals/buffbench/gen-evals.ts:134–261

({
  repoUrl,
  commitShas,
  outputPath,
}: {
  repoUrl: string
  commitShas: string[]
  outputPath?: string
})

}

export async function generateEvalFileV2({
  repoUrl,
  commitShas,
  outputPath,
}: {
  repoUrl: string
  commitShas: string[]
  outputPath?: string
}): Promise<void> {
  const actualRepoName = extractRepoNameFromUrl(repoUrl)

  const client = new CodebuffClient({
    apiKey: process.env[API_KEY_ENV_VAR] || getUserCredentials()?.authToken,
  })

  const finalOutputPath =
    outputPath || path.join(__dirname, `eval-${actualRepoName}-v2.json`)
  const partialOutputPath = finalOutputPath.replace(/\.json$/, '.partial.json')

  console.log(`Processing ${commitShas.length} commits in parallel...`)
  console.log(`Partial results will be saved to: ${partialOutputPath}`)
  console.log(`Final results will be saved to: ${finalOutputPath}\n`)

  const BATCH_SIZE = 5
  const evalCommits: EvalCommitV2[] = []

  const processCommit = async (
    commitSha: string,
  ): Promise<EvalCommitV2 | null> => {
    console.log(`Processing commit ${commitSha.slice(0, 8)}...`)

    return await withTestRepoAndParent(
      {
        repoUrl,
        commitSha,
        initCommand: undefined,
      },
      async (repoPath, commitSha, parentSha) => {
        const fileDiffs = await extractFileDiffsFromCommit(
          repoPath,
          commitSha,
          parentSha,
        )

        const fullDiff = getFullDiff(repoPath, commitSha, parentSha)
        const commitMessage = getCommitMessage(repoPath, commitSha)
        const editedFilePaths = fileDiffs.map((f) => f.path)

        console.log(`Generating eval task for ${commitSha.slice(0, 8)}...`)
        const taskResult = await generateEvalTask({
          client,
          input: {
            commitSha,
            parentSha,
            diff: fullDiff,
            editedFilePaths,
            commitMessage,
            repoPath,