Function countChatHistoryItemTokens

extensions/cli/src/util/tokenizer.ts:106–170

(
  historyItem: ChatHistoryItem,
  model: ModelConfig,
)

 * @returns The estimated token count
 */
export function countChatHistoryItemTokens(
  historyItem: ChatHistoryItem,
  model: ModelConfig,
): number {
  try {
    let tokenCount = 0;

    const message = historyItem.message;

    // Count tokens in content
    tokenCount += countContentTokens(message.content, model);

    // Add tokens for role (roughly 1-2 tokens)
    tokenCount += 2;

    // Add tokens for tool calls if present
    // Skip if toolCallStates exists to avoid double-counting (toolCallStates includes the tool calls)
    if (
      "toolCalls" in message &&
      message.toolCalls &&
      !historyItem.toolCallStates
    ) {
      for (const toolCall of message.toolCalls) {
        tokenCount += countToolCallFunctionTokens(toolCall.function);
      }
    }

    // Add tokens for tool call results
    if (message.role === "tool" && "content" in message) {
      tokenCount += 5; // Tool message structure overhead
    }

    // Add tokens for context items
    for (const contextItem of historyItem.contextItems) {
      tokenCount += encode(contextItem.content).length;
      tokenCount += encode(contextItem.name).length;
      tokenCount += 5; // Context item structure overhead
    }

    // Add tokens for tool call states (tool results/outputs)
    if (historyItem.toolCallStates) {
      for (const toolState of historyItem.toolCallStates) {
        // Count tokens in tool call function (name + arguments)
        tokenCount += countToolCallFunctionTokens(toolState.toolCall?.function);
        // Count tokens in tool outputs (can be very large - thousands of tokens)
        tokenCount += countToolOutputTokens(toolState.output);
      }
    }

    return tokenCount;
  } catch (error) {
    logger.error("Error counting tokens for history item", error);
    // Return a rough estimate based on character count
    const message = historyItem.message;
    const content =
      typeof message.content === "string"
        ? message.content
        : JSON.stringify(message.content);