Function getTokenCount

lib/utils.ts:131–141

(prompt: ChatGPTMessage[] | string)

}

export function getTokenCount(prompt: ChatGPTMessage[] | string): number {
  let chatGptMessages: ChatMessage[];
  if (typeof prompt === "string") {
    chatGptMessages = [{ role: "user", content: prompt }];
  } else {
    chatGptMessages = prompt as ChatMessage[];
  }
  // Per the docs, the tokenizer should be the same for 3.5-turbo and 4.
  const encoded = tokenizer.encodeChat(chatGptMessages, "gpt-4");
  return encoded.length;
}

export function chunkString(
  text: string,