Function runBenchmark

plugins/plugin-eval/src/core/benchmark.js:490–681

(targetPath, options = {})

}

export async function runBenchmark(targetPath, options = {}) {
  const target = await resolveTarget(targetPath);
  const { config, configPath, source } = await loadBenchmarkConfig(target, options);
  const scenarios = (config.scenarios || []).map(normalizeScenario).filter((scenario) => scenario.userInput);
  if (scenarios.length === 0) {
    throw new Error("Benchmark config does not contain any runnable scenarios.");
  }
  if (options.dryRun) {
    throw new Error("CLI-only benchmarking no longer supports --dry-run. Edit the benchmark config, then run `plugin-eval benchmark` for a real Codex execution.");
  }

  const processRunner = options.processRunner || runProcessCapture;
  const codexExecutable = options.codexExecutable || process.env.PLUGIN_EVAL_CODEX_EXECUTABLE || "codex";
  const runId = createRunId();
  const runDirectory = path.join(benchmarkRunsDirectoryForTarget(target), runId);
  await fs.mkdir(runDirectory, { recursive: true });

  let codexVersion = "unknown";
  try {
    const versionResult = await processRunner({
      kind: "codex-version",
      command: codexExecutable,
      args: ["--version"],
      cwd: process.cwd(),
      env: process.env,
      stdoutPath: path.join(runDirectory, "codex-version.stdout.log"),
      stderrPath: path.join(runDirectory, "codex-version.stderr.log"),
    });
    codexVersion = (versionResult.stdoutText || versionResult.stderrText || "unknown").trim().split(/\r?\n/).pop() || "unknown";
  } catch {
    codexVersion = "unknown";
  }

  const usageLines = [];
  const runScenarios = [];

  for (let index = 0; index < scenarios.length; index += 1) {
    const scenario = scenarios[index];
    const scenarioDirectory = path.join(runDirectory, `${String(index + 1).padStart(2, "0")}-${scenario.id}`);
    await fs.mkdir(scenarioDirectory, { recursive: true });

    const provisioned = await provisionBenchmarkWorkspace({
      target,
      config,
      scenarioId: scenario.id,
    });
    const beforeSnapshot = await snapshotWorkspace(provisioned.workspacePath);
    const stdoutPath = path.join(scenarioDirectory, "codex.stdout.jsonl");
    const stderrPath = path.join(scenarioDirectory, "codex.stderr.log");
    const finalMessagePath = path.join(scenarioDirectory, "final-message.txt");

    const args = buildCodexExecArgs({
      config,
      model: options.model || config.runner.model || defaultModelForTarget(target),
      workspacePath: provisioned.workspacePath,
      finalMessagePath,
      prompt: scenario.userInput,
    });