Function countChatHistoryTokens

extensions/cli/src/util/tokenizer.ts:177–191

(
  chatHistory: ChatHistoryItem[],
  model: ModelConfig,
)

 * @returns The estimated total token count
 */
export function countChatHistoryTokens(
  chatHistory: ChatHistoryItem[],
  model: ModelConfig,
): number {
  let totalTokens = 0;

  for (const historyItem of chatHistory) {
    totalTokens += countChatHistoryItemTokens(historyItem, model);
  }

  // Add some overhead for message structure (roughly 3 tokens per message)
  totalTokens += chatHistory.length * 3;

  return totalTokens;
}

/**
 * Calculate the percentage of context used