Function createObservedUsageChecks

plugins/plugin-eval/src/core/observed-usage.js:182–238

(summary)

}

function createObservedUsageChecks(summary) {
  const checks = [];

  if (summary.sampleCount < 3) {
    checks.push(
      createCheck({
        id: "observed-usage-small-sample",
        category: "measurement",
        severity: "warning",
        status: "warn",
        message: "Observed usage coverage is too small to trust as a stable benchmark yet.",
        evidence: [`Samples collected: ${summary.sampleCount}`],
        remediation: ["Capture at least 5 to 10 representative sessions before treating observed usage as a baseline."],
      }),
    );
  }

  const comparison = summary.estimateComparison;
  if (comparison) {
    if (comparison.band === "drift" || comparison.band === "wide-drift") {
      checks.push(
        createCheck({
          id: "observed-usage-estimate-drift",
          category: "budget",
          severity: comparison.band === "wide-drift" ? "error" : "warning",
          status: comparison.band === "wide-drift" ? "fail" : "warn",
          message: "Static budget estimates differ meaningfully from observed input token usage.",
          evidence: [
            `Estimated active tokens: ${comparison.estimatedActiveTokens}`,
            `Observed average input tokens: ${comparison.observedAverageInputTokens}`,
            `Delta ratio: ${round(comparison.deltaRatio * 100)}%`,
          ],
          remediation: [
            "Trim repeated instructions or supporting text if the observed value is higher than expected.",
            "If the static estimate is intentionally conservative, record that assumption in the skill or plugin references.",
          ],
        }),
      );
    }

    if (summary.cachedTokens.average > 0) {
      checks.push(
        createCheck({
          id: "observed-usage-cache-present",
          category: "measurement",
          severity: "info",
          status: "info",
          message: "Observed runs include cached tokens, so repeated sessions are cheaper than the cold-start estimate.",
          evidence: [`Average cached tokens: ${summary.cachedTokens.average}`],
          remediation: ["Track cold-start and warm-cache sessions separately if you need tighter budgeting."],
        }),
      );
    }
  }

  return checks;
}