Function resolveEncoding

src/node/utils/main/tokenizer.ts:121–139

(modelName: ModelName)

}

function resolveEncoding(modelName: ModelName): Promise<string> {
  let promise = encodingPromises.get(modelName);
  if (!promise) {
    promise = run<string>("encodingName", modelName)
      .then((result: unknown) => {
        assert(
          typeof result === "string" && result.length > 0,
          "Token encoding name must be a non-empty string"
        );
        return result;
      })
      .catch((error) => {
        encodingPromises.delete(modelName);
        throw error;
      });
    encodingPromises.set(modelName, promise);
  }
  return promise;
}

function buildCacheKey(modelName: ModelName, text: string): string {
  const checksum = CRC32.str(text);