Function analyzeObservedUsage

plugins/plugin-eval/src/core/observed-usage.js:240–394

(usagePaths = [], rawBudget, target)

}

export async function analyzeObservedUsage(usagePaths = [], rawBudget, target) {
  if (!Array.isArray(usagePaths) || usagePaths.length === 0) {
    return null;
  }

  const snapshots = [];
  for (const usagePath of usagePaths) {
    const loaded = await loadSnapshotsFromFile(usagePath);
    snapshots.push(...loaded);
  }

  if (snapshots.length === 0) {
    throw new Error("Observed usage files were provided, but no usage payloads could be parsed.");
  }

  const inputTokens = buildStat(snapshots.map((snapshot) => snapshot.inputTokens));
  const outputTokens = buildStat(snapshots.map((snapshot) => snapshot.outputTokens));
  const totalTokens = buildStat(snapshots.map((snapshot) => snapshot.totalTokens));
  const cachedTokens = buildStat(snapshots.map((snapshot) => snapshot.cachedTokens));
  const reasoningTokens = buildStat(snapshots.map((snapshot) => snapshot.reasoningTokens));
  const estimatedActiveTokens = rawBudget.trigger_cost_tokens.value + rawBudget.invoke_cost_tokens.value;
  const observedAverageInputTokens = inputTokens.average;
  const deltaTokens = round(observedAverageInputTokens - estimatedActiveTokens);
  const deltaRatio =
    estimatedActiveTokens > 0 ? round(Math.abs(deltaTokens) / estimatedActiveTokens) : 0;

  const observedUsage = {
    method: "observed-usage-files",
    target: {
      name: target.name,
      kind: target.kind,
      path: target.path,
      relativePath: relativePath(process.cwd(), target.path),
    },
    sampleCount: snapshots.length,
    files: [...new Set(snapshots.map((snapshot) => relativePath(process.cwd(), snapshot.sourcePath)))],
    inputTokens,
    outputTokens,
    totalTokens,
    cachedTokens,
    reasoningTokens,
    estimateComparison: estimatedActiveTokens
      ? {
          estimatedActiveTokens,
          observedAverageInputTokens,
          deltaTokens,
          deltaRatio,
          band: classifyEstimateAlignment(deltaRatio),
        }
      : null,
    samples: snapshots.map((snapshot) => ({
      ...snapshot,
      sourcePath: relativePath(process.cwd(), snapshot.sourcePath),
    })),
  };

  const metrics = [
    createMetric({