Function tokenCount

src/utils/tokenCount.ts:4–13

(content: string)

import { Tiktoken } from '@dqbd/tiktoken/lite';

export function tokenCount(content: string): number {
  const encoding = new Tiktoken(
    cl100k_base.bpe_ranks,
    cl100k_base.special_tokens,
    cl100k_base.pat_str
  );
  const tokens = encoding.encode(content);
  encoding.free();
  return tokens.length;
}