Function evaluateClassifier

research/eval_baselines.ts:295–351

(
  name: string,
  classifyFn: (msg: string) => ContextType,
  nRuns: number = 1,  // >1 for stochastic classifiers
)

}

function evaluateClassifier(
  name: string,
  classifyFn: (msg: string) => ContextType,
  nRuns: number = 1,  // >1 for stochastic classifiers
): ClassifierResult {
  // For stochastic classifiers, run multiple times and average
  const allCorrect: boolean[] = []

  // Accumulate per-class counts across all runs
  const tpAccum: Record<string, number> = {}
  const fpAccum: Record<string, number> = {}
  const fnAccum: Record<string, number> = {}
  for (const ctx of CONTEXT_TYPES) {
    tpAccum[ctx] = 0; fpAccum[ctx] = 0; fnAccum[ctx] = 0
  }

  for (let run = 0; run < nRuns; run++) {
    for (const test of TEST_CASES) {
      const predicted = classifyFn(test.message)
      const correct = predicted === test.expectedContext
      allCorrect.push(correct)

      if (correct) {
        tpAccum[test.expectedContext]++
      } else {
        fpAccum[predicted]++
        fnAccum[test.expectedContext]++
      }
    }
  }

  // Bootstrap CI on accuracy
  const ci = bootstrapCI(allCorrect)

  // Per-class metrics (averaged over runs)
  const perClass: Record<string, { precision: number; recall: number; f1: number }> = {}
  for (const ctx of CONTEXT_TYPES) {
    const tp = tpAccum[ctx] / nRuns
    const fp = fpAccum[ctx] / nRuns
    const fn = fnAccum[ctx] / nRuns
    const precision = tp + fp > 0 ? tp / (tp + fp) : 0
    const recall = tp + fn > 0 ? tp / (tp + fn) : 0
    const f1 = precision + recall > 0 ? 2 * precision * recall / (precision + recall) : 0
    perClass[ctx] = { precision, recall, f1 }
  }

  const macroF1 = CONTEXT_TYPES.reduce((sum, ctx) => sum + perClass[ctx].f1, 0) / CONTEXT_TYPES.length

  return {
    name,
    accuracy: ci.mean,
    ci_lower: ci.lower,
    ci_upper: ci.upper,
    perClass,
    macroF1,
  }
}