Function Factuality

evals/shared.ts:81–112

(args: {
  input: string;
  output: string;
  expected?: string;
})

// Wrapper that retries the Factuality scorer on failure
export const Factuality = async (args: {
  input: string;
  output: string;
  expected?: string;
}): Promise<{ name: string; score: number | null; metadata?: Record<string, unknown> }> => {
  let lastError: Error | null = null;

  for (let attempt = 1; attempt <= MAX_SCORER_RETRIES; attempt++) {
    try {
      const result = await FactualityBaseScorer({ ...args, maxTokens: 10_000 });
      // If score is null, treat as failure and retry
      if (result.score === null) {
        lastError = new Error('Scorer returned null score');
        continue;
      }
      return result;
    } catch (error) {
      lastError = error instanceof Error ? error : new Error(String(error));
      // Continue to next attempt
    }
  }

  // All retries exhausted - return error result
  return {
    name: 'Factuality',
    score: null,
    metadata: {
      error: lastError?.message || 'Unknown error after retries',
      retries: MAX_SCORER_RETRIES,
    },
  };
};

import { runAgentInWorker } from '../src/agents/run-in-worker.js';
import { defaultErrorScoreHandler, type Span } from 'braintrust';