Function judgeLatestSession

packages/ai-code-mode/models-eval/run-eval.ts:509–576

(rejudge: boolean)

}

async function judgeLatestSession(rejudge: boolean): Promise<void> {
  const files = listRunLogFiles()
  const payloads = files
    .map((path) => ({ path, payload: readRunLog(path) }))
    .filter(
      (entry): entry is { path: string; payload: RunLogPayload } =>
        !!entry.payload,
    )

  if (payloads.length === 0) {
    console.error('[models-eval] No run logs found to judge.')
    process.exit(1)
  }

  const latestSession = payloads
    .map((entry) => entry.payload.sessionStartedAt)
    .sort()
    .at(-1)
  if (!latestSession) {
    console.error('[models-eval] Could not determine latest session.')
    process.exit(1)
  }

  const sessionPayloads = payloads.filter(
    (entry) => entry.payload.sessionStartedAt === latestSession,
  )
  const rows: Array<EvalRow> = []

  for (const entry of sessionPayloads) {
    const payload = entry.payload
    if (payload.error) {
      rows.push(toEvalRow(payload))
      continue
    }
    if (payload.judge && !rejudge) {
      rows.push(toEvalRow(payload))
      continue
    }

    const computed = computeMetrics(payload.messages)
    const judged = await judgeReports({
      query: payload.query,
      goldReport: payload.goldReport,
      candidateReport: payload.candidateReport,
      typescriptEvidence: formatTypescriptEvidence(computed.typeScriptAttempts),
    })
    payload.judge = judged
    payload.noJudge = false

    writeFileSync(entry.path, `${JSON.stringify(payload, null, 2)}\n`, 'utf8')
    rows.push(toEvalRow(payload))
    console.log(`  ✓ judged from log: ${payload.entry.model}`)
  }

  finalizeRows(
    rows,
    payloads.map((p) => p.payload),
  )