Method create

test/factories/evalFactory.ts:20–92

(options?: CreateEvalOptions)

export default class EvalFactory {
  static async create(options?: CreateEvalOptions) {
    const eval_ = await Eval.create(
      {
        providers: [{ id: 'test-provider' }],
        prompts: ['What is the capital of {{state}}?'],
        tests: [
          { vars: { state: 'colorado' }, assert: [{ type: 'contains', value: 'Denver' }] },
          { vars: { state: 'california' }, assert: [{ type: 'contains', value: 'Sacramento' }] },
        ],
      },
      [
        { raw: 'What is the capital of california?', label: 'What is the capital of {{state}}?' },
        { raw: 'What is the capital of colorado?', label: 'What is the capital of {{state}}?' },
      ],
      { id: randomUUID() },
    );
    await eval_.addPrompts([
      {
        raw: 'What is the capital of california?',
        label: 'What is the capital of {{state}}?',
        provider: 'test-provider',
        metrics: {
          score: 1,
          testPassCount: 1,
          testFailCount: 1,
          testErrorCount: 0,
          assertPassCount: 1,
          assertFailCount: 1,
          totalLatencyMs: 200,
          tokenUsage: { total: 20, prompt: 10, completion: 10, cached: 0 },
          namedScores: {},
          namedScoresCount: {},
          redteam: {
            pluginPassCount: {},
            pluginFailCount: {},
            strategyPassCount: {},
            strategyFailCount: {},
          },
          cost: 0.007,
        },
      },
    ]);

    // If no options are provided, create the default test results
    if (!options || options.numResults === undefined) {
      await this.addDefaultResults(eval_);
      return eval_;
    }

    // Generate the specified number of test results with the requested characteristics
    const numResults = options.numResults === 0 ? 0 : options.numResults || 2;
    const resultTypes = options.resultTypes || ['success'];
    const withHighlights = options.withHighlights || false;
    const withNamedScores = options.withNamedScores || false;
    const searchableContent = options.searchableContent || '';

    for (let i = 0; i < numResults; i++) {
      // Cycle through the result types