Function splitMarkdownIntoSentences

lib/embed-docs/utils.ts:199–234

(markdown: string)

}

export function splitMarkdownIntoSentences(markdown: string): string[] {
  /** Does what it says on the tin.
   *
   * Splits a paragraph of markdown (since we previously split by newline) into
   * sentences. *Does not* split by newline. **/
  let tmpMarkedItems: string[] = [];
  let tmpString = markdown;

  const markdownUrlRegex = /(!\[[\w\s\W]+]\([\w.\-\/]+\))/g;
  let result;
  while ((result = markdownUrlRegex.exec(markdown))) {
    tmpMarkedItems.push(result[0]);
    tmpString = tmpString.replace(result[0], `{${tmpMarkedItems.length - 1}}`);
  }

  let sentences = tmpString
    .split(/(?<=[.!?])\s/g)
    .map((e) => e.trim())
    .filter(Boolean);

  sentences = sentences.map((sentence) => {
    const placeholderRegex = /\{(\d+)}/g;
    let result;
    while ((result = placeholderRegex.exec(sentence))) {
      sentence = sentence.replace(result[0], tmpMarkedItems[Number(result[1])]);
    }
    return sentence + " ";
  });

  // Remove the space at the end of the last sentence
  sentences[sentences.length - 1] = sentences[sentences.length - 1].slice(
    0,
    -1,
  );
  return sentences;
}

export function joinShortChunks(arr: string[], tokenLimit: number): string[] {
  const result = [];