Function runAgentBenchmark

benchmark/simple-benchmark.ts:188–259

(options: BenchmarkOptions)

}

async function runAgentBenchmark(options: BenchmarkOptions): Promise<BenchmarkResult> {
  const toolUsage: ToolUsage = {};
  let imageTokens = 0;

  const q = query({
    prompt: options.task,
    options: {
      systemPrompt: options.systemPrompt,
      mcpServers: options.mcpServers,
      disallowedTools: options.disallowedTools,
      permissionMode: "bypassPermissions",
      allowDangerouslySkipPermissions: true,
    },
  });

  let resultMessage: SDKMessage | null = null;
  for await (const message of q) {
    if (message.type === "assistant") {
      for (const block of message.message.content) {
        if (block.type === "tool_use") {
          const toolName = block.name;
          toolUsage[toolName] = (toolUsage[toolName] || 0) + 1;
        }
      }
    }

    if (message.type === "user" && "tool_use_result" in message && message.tool_use_result) {
      const result = message.tool_use_result as {
        content?: Array<{ type: string; data?: string; mimeType?: string }>;
      };
      if (result.content) {
        for (const item of result.content) {
          if (item.type === "image" && item.data && item.mimeType) {
            const tokens = calculateImageTokensFromBase64(
              item.data,
              item.mimeType as ImageMimeType
            );
            if (tokens) imageTokens += tokens;
          }
        }
      }
    }

    if (message.type === "result") {
      resultMessage = message;
    }
  }

  if (!resultMessage || resultMessage.type !== "result") {
    return {
      success: false,
      inputTokens: 0,
      outputTokens: 0,
      imageTokens: 0,
      totalCostUsd: 0,
      durationMs: 0,
      numTurns: 0,
      toolUsage,