Function countTotalInputTokens

extensions/cli/src/util/tokenizer.ts:313–335

(params: TotalInputTokenParams)

 * @returns Total estimated input token count
 */
export function countTotalInputTokens(params: TotalInputTokenParams): number {
  const { chatHistory, systemMessage, tools } = params;

  let totalTokens = countChatHistoryTokens(chatHistory, params.model);

  // Add system message tokens if provided and not already in history
  if (systemMessage) {
    const hasSystemInHistory = chatHistory.some(
      (item) => item.message.role === "system",
    );
    if (!hasSystemInHistory) {
      totalTokens += encode(systemMessage).length;
      totalTokens += 4; // Message structure overhead (role + formatting)
    }
  }

  // Add tool definition tokens
  if (tools && tools.length > 0) {
    totalTokens += countToolDefinitionTokens(tools);
  }

  return totalTokens;
}

/**
 * Parameters for context length validation