Function chunkString

lib/utils.ts:143–166

(
  text: string,
  chunkSize: number,
  overlap: number,
)

}

export function chunkString(
  text: string,
  chunkSize: number,
  overlap: number,
): string[] {
  /** Splits a long string into chunks of length `chunkSize`, with overlap `overlap`
   * Useful for splitting up long strings for GPT. **/
  const encoding = tokenizer.encode(text);
  console.log("Chunking string. Number of tokens = " + encoding.length);

  const chunks = [];
  let i = 0;
  let start,
    end = 0;
  while (end < encoding.length) {
    start = i * (chunkSize - overlap);
    end = start + chunkSize;

    chunks.push(tokenizer.decode(encoding.slice(start, end)));
    i++;
  }
  console.log("Outputting " + chunks.length + " chunks of length " + chunkSize);
  return chunks;
}

export function objectNotEmpty(obj: Object): boolean {
  return Object.keys(obj).length > 0;