Function countTokensInternal

src/node/utils/main/tokenizer.ts:146–178

(modelName: ModelName, text: string)

}

async function countTokensInternal(modelName: ModelName, text: string): Promise<number> {
  assert(typeof text === "string", "Tokenizer countTokens expects string input");
  if (text.length === 0) {
    return 0;
  }

  const key = buildCacheKey(modelName, text);
  const cached = tokenCountCache.get(key);
  if (cached !== undefined) {
    return cached;
  }

  let pending = inFlightCounts.get(key);
  if (!pending) {
    const payload: CountTokensInput = { modelName, input: text };
    pending = run<number>("countTokens", payload)
      .then((value: unknown) => {
        assert(
          typeof value === "number" && Number.isFinite(value) && value >= 0,
          "Tokenizer must return a non-negative finite token count"
        );
        tokenCountCache.set(key, value);
        inFlightCounts.delete(key);
        return value;
      })
      .catch((error) => {
        inFlightCounts.delete(key);
        throw error;
      });
    inFlightCounts.set(key, pending);
  }
  return pending;
}

export function loadTokenizerModules(
  modelsToWarm: string[] = Array.from(DEFAULT_WARM_MODELS)