Function runEvalInternal

src/evaluator.ts:1399–1596

({
  provider,
  prompt, // raw prompt
  test,
  testSuite,
  delay,
  nunjucksFilters: filters,
  evaluateOptions,
  // TODO(ian): Rename these public `Idx` fields to `Index` with compatibility handling.
  testIdx: testIndex,
  promptIdx: promptIndex,
  repeatIndex,
  conversations,
  registers,
  isRedteam,
  abortSignal,
  deferGrading,
  evalId,
  providerCallQueue,
  rateLimitRegistry,
}: RunEvalOptions)

}

async function runEvalInternal({
  provider,
  prompt, // raw prompt
  test,
  testSuite,
  delay,
  nunjucksFilters: filters,
  evaluateOptions,
  // TODO(ian): Rename these public `Idx` fields to `Index` with compatibility handling.
  testIdx: testIndex,
  promptIdx: promptIndex,
  repeatIndex,
  conversations,
  registers,
  isRedteam,
  abortSignal,
  deferGrading,
  evalId,
  providerCallQueue,
  rateLimitRegistry,
}: RunEvalOptions): Promise<EvaluateResult[]> {
  provider.delay ??= delay ?? getEnvInt('PROMPTFOO_DELAY_MS', 0);
  invariant(
    typeof provider.delay === 'number',
    `Provider delay should be set for ${provider.label}`,
  );

  const state = createRunEvalState({ provider, prompt, test });
  attachConversationVar({
    conversations,
    conversationKey: state.conversationKey,
    prompt,
    test,
    vars: state.vars,
  });
  Object.assign(state.vars, registers);
  Object.assign(
    state.vars,
    getEvalRuntimeVars({
      evalId,
      promptIndex,
      repeatIndex,
      testIndex,
    }),
  );

  let setup = state.setup;
  let latencyMs = 0;
  let traceContext: Awaited<ReturnType<typeof generateTraceContextIfNeeded>> | undefined;

  try {
    const rendered = await renderRunEvalPrompt({
      filters,
      isRedteam,
      provider,
      promptForRender: state.promptForRender,
      test,
      testSuite,