Function createStarterBenchmarkConfig

plugins/plugin-eval/src/core/benchmark.js:179–216

(target, options = {})

}

async function createStarterBenchmarkConfig(target, options = {}) {
  if (!["skill", "plugin"].includes(target.kind)) {
    throw new Error("Benchmarking only supports Codex skills and plugins.");
  }

  return {
    kind: "plugin-eval-benchmark",
    schemaVersion: BENCHMARK_SCHEMA_VERSION,
    version: BENCHMARK_SCHEMA_VERSION,
    targetKind: target.kind,
    targetName: target.name,
    runner: {
      type: "codex-cli",
      model: options.model || defaultModelForTarget(target),
      sandbox: "workspace-write",
      approvalPolicy: "never",
      extraArgs: [],
    },
    workspace: {
      sourcePath: path.resolve(options.sourcePath || process.cwd()),
      setupMode: "copy",
      preserve: "on-failure",
    },
    targetProvisioning: {
      mode: defaultTargetProvisioningMode(target),
    },
    verifiers: {
      commands: [],
    },
    notes: [
      "Edit workspace.sourcePath so it points at the repo or template you want Codex to work inside.",
      "Edit the scenarios so they match real tasks instead of generic starter prompts.",
      "Benchmark means real codex exec runs now. There is no simulated dry-run mode.",
    ],
    setupQuestions: buildSetupQuestions(target),
    scenarios: target.kind === "plugin" ? buildPluginScenarios(target) : buildSkillScenarios(target),
  };
}

async function loadBenchmarkConfig(target, options = {}) {
  if (options.configPath) {