Function heuristicTokens

src/utils/tokenizer.ts:44–63

(text: string)

 * tracks real BPE much better than a flat divisor without needing the vocab.
 */
export function heuristicTokens(text: string): number {
  if (!text) return 0;
  // Count alphanumeric runs (≈ 1-2 tokens each for long words) + standalone
  // punctuation/symbols (≈ 1 token each) + whitespace runs (cheap).
  let tokens = 0;
  const wordRuns = text.match(/[A-Za-z0-9_]+/g);
  if (wordRuns) {
    for (const w of wordRuns) {
      // BPE splits long identifiers; ~4 chars per sub-token is a good code average.
      tokens += Math.max(1, Math.ceil(w.length / 4));
    }
  }
  // Non-word, non-space chars (operators, brackets, punctuation) ≈ 1 token each.
  const punct = text.match(/[^\sA-Za-z0-9_]/g);
  if (punct) tokens += punct.length;
  // Newlines are usually their own token in code.
  const newlines = text.match(/\n/g);
  if (newlines) tokens += Math.ceil(newlines.length * 0.5);
  return Math.max(1, tokens);
}

/** Kick off the lazy load of gpt-tokenizer. Safe to call repeatedly. */
async function ensureEncoder(): Promise<void> {