Function loadTokenizerModules

src/node/utils/main/tokenizer.ts:180–201

(
  modelsToWarm: string[] = Array.from(DEFAULT_WARM_MODELS)
)

}

export function loadTokenizerModules(
  modelsToWarm: string[] = Array.from(DEFAULT_WARM_MODELS)
): Promise<Array<PromiseSettledResult<string>>> {
  if (shouldUseApproxTokenizer()) {
    const fulfilled: Array<PromiseFulfilledResult<string>> = modelsToWarm.map(() => ({
      status: "fulfilled",
      value: APPROX_ENCODING,
    }));
    return Promise.resolve(fulfilled);
  }

  return Promise.allSettled(
    modelsToWarm.map((modelString) => {
      const modelName = normalizeModelKey(modelString);
      // Skip unknown models during warmup
      if (!modelName) {
        return Promise.reject(new Error(`Unknown model: ${modelString}`));
      }
      return resolveEncoding(modelName);
    })
  );
}

export async function getTokenizerForModel(
  modelString: string,