Function countLlamaInputTokens

app/src/utils/countTokens.ts:47–48

(input: Parameters<typeof serializeChatInput>[0])

export const countLlamaTokens = (input: string) => llamaTokenizer.encode(input).length;

export const countLlamaInputTokens = (input: Parameters<typeof serializeChatInput>[0]) =>
  countLlamaTokens(serializeChatInput(input, { pipelineVersion: CURRENT_PIPELINE_VERSION }));

export const countLlamaOutputTokens = (output: ChatCompletionMessage) =>
  countLlamaTokens(serializeChatOutput(output));