Function main

packages/ai-code-mode/models-eval/run-eval.ts:731–958

()

}

async function main() {
  const {
    ollamaOnly,
    noJudge,
    captureOnly,
    judgeLatest,
    rejudge,
    modelFilter,
  } = parseArgs(process.argv.slice(2))

  mkdirSync(LOG_DIR, { recursive: true })
  if (judgeLatest) {
    await judgeLatestSession(rejudge)
    return
  }

  const effectiveNoJudge = noJudge || captureOnly
  const sessionStartedAt = new Date().toISOString()
  console.log(
    `[models-eval] Run logs → ${relative(process.cwd(), LOG_DIR) || LOG_DIR} (gitignored)`,
  )

  if (!effectiveNoJudge && !process.env.ANTHROPIC_API_KEY) {
    console.warn(
      '[models-eval] ANTHROPIC_API_KEY is not set; judging will fail. Use --no-judge to skip, or set the key.',
    )
  }

  const gold = loadGoldFile()
  const query = gold.prompt
  const goldReport = extractReportTextFromGold(gold.messages)

  const { tool, systemPrompt } = getCodeModeTools()
  const tools = [tool, getSchemaInfoTool]
  const systemPrompts = [DATABASE_DEMO_SYSTEM_PROMPT, systemPrompt]

  let models = ollamaOnly
    ? EVAL_MODELS.filter((m) => m.model.startsWith('ollama:'))
    : EVAL_MODELS

  if (modelFilter && modelFilter.size > 0) {
    models = models.filter((m) => modelFilter.has(m.model))
    if (models.length === 0) {
      console.error(
        '[models-eval] No models matched --models filter. Check eval-config model ids.',
      )
      process.exit(1)
    }
  }

  const rows: Array<EvalRow> = []

  for (const entry of models) {
    const { provider, modelId } = parseModelId(entry.model)
    const modelCategory = getModelCategory(entry.model)
    console.log(`\n→ Running: ${entry.name} (${entry.model})`)
    const adapter = getTextAdapter(provider, modelId)
    const t0 = Date.now()