Function appendRunEvalOptionsForTestCase

src/evaluator.ts:2353–2415

({
  concurrency,
  conversations,
  evalId,
  nextTestIdx,
  options,
  promptIdCache,
  promptIndexMap,
  providerAbortSignal,
  rateLimitRegistry,
  registers,
  runEvalOptions,
  testCase,
  testSuite,
}: {
  concurrency: number;
  conversations: EvalConversations;
  evalId: string;
  nextTestIdx: number;
  options: InternalEvaluateOptions;
  promptIdCache: Map<Prompt, string>;
  promptIndexMap: Map<string, number>;
  providerAbortSignal?: AbortSignal;
  rateLimitRegistry?: RateLimitRegistryRef;
  registers: EvalRegisters;
  runEvalOptions: RunEvalOptions[];
  testCase: AtomicTestCase;
  testSuite: TestSuite;
})

}

function appendRunEvalOptionsForTestCase({
  concurrency,
  conversations,
  evalId,
  nextTestIdx,
  options,
  promptIdCache,
  promptIndexMap,
  providerAbortSignal,
  rateLimitRegistry,
  registers,
  runEvalOptions,
  testCase,
  testSuite,
}: {
  concurrency: number;
  conversations: EvalConversations;
  evalId: string;
  nextTestIdx: number;
  options: InternalEvaluateOptions;
  promptIdCache: Map<Prompt, string>;
  promptIndexMap: Map<string, number>;
  providerAbortSignal?: AbortSignal;
  rateLimitRegistry?: RateLimitRegistryRef;
  registers: EvalRegisters;
  runEvalOptions: RunEvalOptions[];
  testCase: AtomicTestCase;
  testSuite: TestSuite;
}) {
  const promptPrefix = testCase.options?.prefix || getDefaultTest(testSuite)?.options?.prefix || '';
  const promptSuffix = testCase.options?.suffix || getDefaultTest(testSuite)?.options?.suffix || '';
  const varCombinations =
    getEnvBool('PROMPTFOO_DISABLE_VAR_EXPANSION') || testCase.options?.disableVarExpansion
      ? [testCase.vars]
      : generateVarCombinations(testCase.vars || {});

  for (let repeatIndex = 0; repeatIndex < (options.repeat || 1); repeatIndex++) {
    for (const vars of varCombinations) {
      appendRunEvalOptionsForVars({
        concurrency,
        conversations,
        evalId,
        options,
        promptIdCache,
        promptIndexMap,
        promptPrefix,
        promptSuffix,
        providerAbortSignal,
        rateLimitRegistry,
        registers,
        repeatIndex,
        runEvalOptions,
        testCase,
        testIdx: nextTestIdx,
        testSuite,
        vars,
      });
      nextTestIdx++;