Function evaluateWithSource

src/evaluate.ts:317–376

(
  testSuite: EvaluateTestSuite,
  options: InternalEvaluateOptions = {},
)

}

export async function evaluateWithSource(
  testSuite: EvaluateTestSuite,
  options: InternalEvaluateOptions = {},
) {
  const { author: suiteAuthor, ...testSuiteConfig } = testSuite;

  if (testSuiteConfig.writeLatestResults) {
    await runDbMigrations();
  }

  const loadedProviders = await loadApiProviders(testSuiteConfig.providers, {
    env: testSuiteConfig.env,
  });
  const providerMap = buildConfiguredProviderMap(loadedProviders);
  const constructedTestSuite = await createRuntimeTestSuite(testSuiteConfig, loadedProviders);
  await resolveNestedProviders(testSuiteConfig, constructedTestSuite, providerMap);

  const parsedProviderPromptMap = readProviderPromptMap(
    testSuiteConfig,
    constructedTestSuite.prompts,
  );
  const unifiedConfig = createSerializableUnifiedConfig(
    testSuiteConfig,
    constructedTestSuite.prompts,
  );
  const author = getAuthor(suiteAuthor);
  const evalRecord = testSuiteConfig.writeLatestResults
    ? await Eval.create(unifiedConfig, constructedTestSuite.prompts, { author })
    : new Eval(unifiedConfig, { author });

  const ret = await cache.withCacheEnabled(options.cache === false ? false : undefined, () =>
    doEvaluate(
      {
        ...constructedTestSuite,
        providerPromptMap: parsedProviderPromptMap,
      },
      evalRecord,
      {
        isRedteam: Boolean(testSuiteConfig.redteam),
        ...options,
      },
    ),
  );

  await maybeShareEval(testSuiteConfig, ret);
  if (testSuiteConfig.outputPath) {
    const outputPaths =
      typeof testSuiteConfig.outputPath === 'string'
        ? [testSuiteConfig.outputPath]
        : testSuiteConfig.outputPath;
    warnOnDegradedJsonlRecovery(evalRecord, outputPaths);
    // writeMultipleOutputs maps each path through writeOutput, so it covers the single-path
    // case too — matching the doEval call site in src/node/doEval.ts.
    if (outputPaths.length) {
      await writeMultipleOutputs(outputPaths, evalRecord, null);
    }
  }