Function make

packages/ai/ai/src/Tokenizer.ts:150–168

(options: {
  /**
   * Function to tokenize a prompt into an array of token numbers.
   */
  readonly tokenize: (
    /**
     * The prompt to tokenize.
     */
    content: Prompt.Prompt
  ) => Effect.Effect<Array<number>, AiError.AiError>
})

 * @category Constructors
 */
export const make = (options: {
  /**
   * Function to tokenize a prompt into an array of token numbers.
   */
  readonly tokenize: (
    /**
     * The prompt to tokenize.
     */
    content: Prompt.Prompt
  ) => Effect.Effect<Array<number>, AiError.AiError>
}): Service =>
  Tokenizer.of({
    tokenize(input) {
      return options.tokenize(Prompt.make(input))
    },
    truncate(input, tokens) {
      return truncate(Prompt.make(input), options.tokenize, tokens)
    }
  })

const truncate = (
  self: Prompt.Prompt,