Function splitIntoTextChunks

lib/embed-docs/utils.ts:66–103

(
  sectionText: string,
  ignoreLines: string[],
)

}

export function splitIntoTextChunks(
  sectionText: string,
  ignoreLines: string[],
): string[] {
  const codeAndNonCodeChunks = splitOutCodeChunks(sectionText);
  return codeAndNonCodeChunks
    .map((codeNonCodeChunk): string[] => {
      if (codeNonCodeChunk.startsWith("```")) {
        return [codeNonCodeChunk];
      }
      return joinShortChunks(
        codeNonCodeChunk
          .split("\n")
          .filter(
            (chunk) =>
              !(
                ignoreLines.includes(RemoveMarkdown(chunk.trim())) ||
                ignoreLines.includes(chunk.trim())
              ),
          )
          // Below aims to remove not-useful lines. E.g. link to privacy policy, today's date etc
          .filter((chunk) => isTextWithSubstance(chunk.trim()))
          // Tabs are more token-efficient than 4 spaces
          .map((chunk) => chunk.replaceAll(/ {4}/g, "\t") + "\n")
          // Remove duplicate lines
          .filter((line, idx, lines) => idx === 0 || line !== lines[idx - 1])
          .map((chunk) => {
            // Break up large paragraphs into sentences
            const chunkTokenCount = getTokenCount(chunk);
            if (chunkTokenCount <= 80) return chunk;
            return splitMarkdownIntoSentences(chunk);
          })
          .flat(),
        25,
      );
    })
    .flat();
}

export function splitOutCodeChunks(sectionText: string): string[] {
  const chunks: string[] = [];