Function countStringTokens

src/modules/chat.ts:201–205

(txt: string)

const enc = getEncoding('gpt2');

export function countStringTokens(txt: string) {
  // Tokenize the content
  const content = enc.encode(txt);
  return content.length;
}

function countChatTokens(
  chatMessages: (