Function getTokenizerForModel

src/node/utils/main/tokenizer.ts:203–219

(
  modelString: string,
  metadataModelOverride?: string
)

}

export async function getTokenizerForModel(
  modelString: string,
  metadataModelOverride?: string
): Promise<Tokenizer> {
  if (shouldUseApproxTokenizer()) {
    return getApproxTokenizer();
  }

  const resolvedModel = metadataModelOverride ?? modelString;
  const modelName = resolveModelName(resolvedModel);
  const encodingName = await resolveEncoding(modelName);

  return {
    encoding: encodingName,
    countTokens: (input: string) => countTokensInternal(modelName, input),
  };
}

export function countTokens(modelString: string, text: string): Promise<number> {
  if (shouldUseApproxTokenizer()) {