Function main

benchmark/simple-benchmark.ts:281–341

()

}

async function main(): Promise<void> {
  cli.header("Simple Benchmark: Screenshots vs WebMCP Tools\nAgent SDK Version");

  cli.section("STARTING DEV SERVER");
  try {
    await startDevServer();
  } catch (err) {
    cli.error(`Failed to start dev server: ${err}`);
    process.exit(1);
  }

  cli.info(`Task: ${TASK}`);
  cli.info(`Runs per approach: ${NUM_RUNS}`);
  cli.info("Using Claude Agent SDK for realistic agentic behavior");
  cli.info("Screenshot approach: WebMCP tools blocked");
  cli.info("WebMCP approach: All tools available, prefers semantic tools");

  const screenshotResults: BenchmarkResult[] = [];
  const webmcpResults: BenchmarkResult[] = [];

  for (let i = 0; i < NUM_RUNS; i++) {
    cli.section(`RUN ${i + 1}/${NUM_RUNS}`);

    cli.info("Running Screenshot approach...");
    const screenshotResult = await runScreenshotBenchmark();
    screenshotResults.push(screenshotResult);
    if (screenshotResult.success) {
      cli.success(
        `Completed: ${screenshotResult.inputTokens.toLocaleString()} in / ${screenshotResult.outputTokens.toLocaleString()} out, $${screenshotResult.totalCostUsd.toFixed(4)}`
      );
    } else {
      cli.error("Failed");
    }

    cli.info("Running WebMCP approach...");
    const webmcpResult = await runWebMCPBenchmark();
    webmcpResults.push(webmcpResult);
    if (webmcpResult.success) {
      cli.success(
        `Completed: ${webmcpResult.inputTokens.toLocaleString()} in / ${webmcpResult.outputTokens.toLocaleString()} out, $${webmcpResult.totalCostUsd.toFixed(4)}`
      );
    } else {
      cli.error("Failed");
    }
  }

  const screenshotAgg = aggregateBenchmarkResults("Screenshot", screenshotResults);
  const webmcpAgg = aggregateBenchmarkResults("WebMCP", webmcpResults);

  cli.section("SCREENSHOT APPROACH RESULTS");
  printAgentApproachResults(screenshotAgg);

  cli.section("WEBMCP APPROACH RESULTS");
  printAgentApproachResults(webmcpAgg);

  printAgentComparisonResults(screenshotAgg, webmcpAgg, "FINAL COMPARISON");
  printAgentCostAnalysis(screenshotAgg, webmcpAgg);