Function updateDerivedMetrics

src/evaluator.ts:1764–1793

(
  metrics: PromptMetrics,
  derivedMetrics: NonNullable<TestSuite['derivedMetrics']>,
  evalStep: RunEvalOptions,
  promptEvalCount: number,
)

}

async function updateDerivedMetrics(
  metrics: PromptMetrics,
  derivedMetrics: NonNullable<TestSuite['derivedMetrics']>,
  evalStep: RunEvalOptions,
  promptEvalCount: number,
) {
  const math = await import('mathjs');
  if (Object.prototype.hasOwnProperty.call(metrics.namedScores, '__count')) {
    logger.warn("Metric name '__count' is reserved for derived metrics and will be overridden.");
  }

  const evalContext: Record<string, number> = {
    ...metrics.namedScores,
    __count: promptEvalCount,
  };
  for (const metric of derivedMetrics) {
    metrics.namedScores[metric.name] ??= 0;
    try {
      metrics.namedScores[metric.name] =
        typeof metric.value === 'function'
          ? metric.value(evalContext, evalStep)
          : math.evaluate(metric.value, evalContext);
      evalContext[metric.name] = metrics.namedScores[metric.name];
    } catch (error) {
      logger.debug(
        `Could not evaluate derived metric '${metric.name}': ${(error as Error).message}`,
      );
    }
  }
}

function updateComparisonReporterTotals({
  ciProgressReporter,