Function buildCacheKey

src/node/utils/main/tokenizer.ts:141–144

(modelName: ModelName, text: string)

}

function buildCacheKey(modelName: ModelName, text: string): string {
  const checksum = CRC32.str(text);
  return `${modelName}:${checksum}:${text.length}`;
}

async function countTokensInternal(modelName: ModelName, text: string): Promise<number> {
  assert(typeof text === "string", "Tokenizer countTokens expects string input");