Function renderBenchmarkRun

plugins/plugin-eval/src/renderers/markdown.js:544–599

(payload)

}

function renderBenchmarkRun(payload) {
  const benchmarkPayload = ensureBenchmarkRunPayload(payload);

  return [
    `# Benchmark Run: ${benchmarkPayload.target.name}`,
    "",
    section("At a Glance", [
      `- Mode: ${benchmarkPayload.mode}`,
      `- Codex version: ${benchmarkPayload.codexVersion}`,
      `- Scenarios: ${benchmarkPayload.config.scenarioCount}`,
      `- Model: ${benchmarkPayload.config.model}`,
      `- Workspace source: \`${benchmarkPayload.config.workspaceSourcePath}\``,
      `- Setup mode: ${benchmarkPayload.config.workspaceSetupMode}`,
      `- Preserve policy: ${benchmarkPayload.config.workspacePreserve}`,
      ...(benchmarkPayload.usageLogPath ? [`- Usage log: \`${benchmarkPayload.usageLogPath}\``] : []),
      `- Run directory: \`${benchmarkPayload.runDirectory}\``,
      `- Usage availability: ${benchmarkPayload.summary.usageAvailability}`,
      `- Average input tokens: ${benchmarkPayload.summary.averageInputTokens}`,
      `- Average output tokens: ${benchmarkPayload.summary.averageOutputTokens}`,
      `- Average total tokens: ${benchmarkPayload.summary.averageTotalTokens}`,
      `- Tool calls: ${benchmarkPayload.summary.toolCallCount}`,
      `- Shell commands: ${benchmarkPayload.summary.shellCommandCount}`,
      `- Failed shell commands: ${benchmarkPayload.summary.failedShellCommands}`,
      `- Generated files: ${benchmarkPayload.summary.generatedFileCount}`,
      `- Generated tests: ${benchmarkPayload.summary.generatedTestFileCount}`,
    ]),
    "",
    section("Why It Matters", [
      "- Benchmarking now runs real `codex exec` sessions in isolated workspaces instead of simulating the skill or plugin through a single API request.",
      `- Completed scenarios: ${benchmarkPayload.summary.completedScenarios}/${benchmarkPayload.summary.scenarioCount}`,
      `- Usage samples collected: ${benchmarkPayload.summary.sampleCount}`,
    ]),
    "",
    section(
      "Fix First",
      benchmarkPayload.summary.failedScenarios > 0
        ? ["- At least one scenario failed. Inspect the scenario logs and any preserved workspace before trusting the benchmark."]
        : benchmarkPayload.summary.usageAvailability === "unavailable"
          ? ["- No usage telemetry was emitted, so treat workspace outcomes and verifier results as the primary benchmark signal."]
          : ["- Re-run analysis with the usage log before you start optimizing the skill or plugin."],
    ),
    "",
    section("Recommended Next Step", renderNextAction(benchmarkPayload.nextAction)),
    "",
    section("Details", [
      detailsBlock(
        "Scenarios",
        benchmarkPayload.scenarios
          .map((scenario) => renderBenchmarkScenario(scenario, benchmarkPayload.mode))
          .join("\n\n"),
      ),
      detailsBlock("Workflow follow-up", benchmarkPayload.nextSteps.map((item) => `- ${item}`).join("\n")),
      detailsBlock("Use From Codex Chat", renderWorkflowGuide(benchmarkPayload.workflowGuide)),
    ]),
  ].join("\n");
}

function renderComparison(payload) {