Function getSupportedModel

app/src/utils/countTokens.ts:36–43

(model: string)

// "gpt-tokens" does not support new models. This is a temporary workaround.
function getSupportedModel(model: string): supportModelType {
  const modelMapping: Record<string, supportModelType> = {
    "gpt-4-0125-preview": "gpt-4-1106-preview",
    "gpt-3.5-turbo-0125": "gpt-3.5-turbo-1106",
  };

  return modelMapping[model] || (model as supportModelType);
}

export const countLlamaTokens = (input: string) => llamaTokenizer.encode(input).length;