Method evaluate

src/evaluator.ts:4735–4848

()

  }

  async evaluate(): Promise<TEvaluation> {
    // Initialize OTEL SDK if tracing is enabled
    // Check env flag, test suite level, and default test metadata
    const tracingEnabled =
      getEnvBool('PROMPTFOO_TRACING_ENABLED', false) ||
      this.testSuite.tracing?.enabled === true ||
      (typeof this.testSuite.defaultTest === 'object' &&
        this.testSuite.defaultTest?.metadata?.tracingEnabled === true) ||
      this.testSuite.tests?.some((t) => t.metadata?.tracingEnabled === true);
    let otelInitialized = false;
    let otlpReceiverAcquired = false;

    let evaluationError: unknown;
    try {
      otlpReceiverAcquired = await startOtlpReceiverIfNeeded(this.testSuite, this.store.id);
      if (tracingEnabled) {
        logger.debug('[Evaluator] Initializing OTEL SDK for tracing');
        const otelConfig = getDefaultOtelConfig();
        initializeOtel(otelConfig);
        otelInitialized = true;
      }

      return await this._runEvaluation();
    } catch (error) {
      evaluationError = error;
      throw error;
    } finally {
      // Close the JSONL writers first, before the (possibly multi-second) OTEL / provider
      // teardown below, so the streamed file is fully flushed before the post-run rewrite
      // reads it back and the file handle is released promptly. allSettled so one writer's
      // close failure neither blocks cleanup nor masks another writer's error.
      const writerCloseResults = await Promise.allSettled(
        this.fileWriters.map((writer) => writer.close()),
      );
      const writerCloseErrors = writerCloseResults.flatMap((result) =>
        result.status === 'rejected' ? [result.reason] : [],
      );

      let cleanupError: unknown;
      try {
        // Flush and shutdown OTEL SDK
        if (otelInitialized) {
          logger.debug('[Evaluator] Flushing OTEL spans...');
          await flushOtel();
          await shutdownOtel();
        }

        if (otlpReceiverAcquired && isOtlpReceiverStarted()) {
          // Add a delay to allow providers to finish exporting spans
          logger.debug('[Evaluator] Waiting for span exports to complete...');
          await sleep(3000);
        }
        await stopOtlpReceiverIfNeeded(otlpReceiverAcquired, this.store.id);

        // Clean up Python worker pools to prevent resource leaks
        await providerRegistry.shutdownAll();

        // Log rate limit metrics for debugging before cleanup