Function countLlamaTokens

app/src/utils/countTokens.ts:45–45

(input: string)

}

export const countLlamaTokens = (input: string) => llamaTokenizer.encode(input).length;

export const countLlamaInputTokens = (input: Parameters<typeof serializeChatInput>[0]) =>
  countLlamaTokens(serializeChatInput(input, { pipelineVersion: CURRENT_PIPELINE_VERSION }));