Function joinShortChunks

lib/embed-docs/utils.ts:236–255

(arr: string[], tokenLimit: number)

}

export function joinShortChunks(arr: string[], tokenLimit: number): string[] {
  const result = [];
  let tempStr = "";
  for (let i = 0; i < arr.length; i++) {
    if (
      getTokenCount(RemoveMarkdown(tempStr + arr[i]).trim()) <=
      tokenLimit + 7 // +7 corrects for chat formatting
    ) {
      tempStr += arr[i];
    } else {
      // Do this in case the final string is too short
      if (tempStr) result.push(tempStr);
      tempStr = arr[i];
    }
    //add the last short string sequence if it exists
    if (i === arr.length - 1 && tempStr) result.push(tempStr);
  }

  return result;
}

export function removeRepetition(str: string): string {
  const regex = /^(.*?)(?:\s)?\1$/;