Function embedDocsFromUrl

lib/embed-docs/embedDocs.ts:19–121

(
  docsUrls: string[],
  orgId: number,
  isDocusaurus: boolean,
  ignoreLines: string[],
)

});

export async function embedDocsFromUrl(
  docsUrls: string[],
  orgId: number,
  isDocusaurus: boolean,
  ignoreLines: string[],
): Promise<DocChunkInsert[]> {
  // Split the links into chunks of 10 to avoid overloading the API
  const docUrlChunks = [];
  for (let i = 0; i < docsUrls.length; i += 10) {
    docUrlChunks.push(docsUrls.slice(i, i + 10));
  }
  // Iterate over the collected links
  const out: DocChunkInsert[][][] = [];
  for (const docUrlChunk of docUrlChunks) {
    out.push(
      await Promise.all(
        docUrlChunk.map(async (url) => {
          console.log("Loading documents from " + url);
          // Initialize an APIReferenceLoader with the option to scrape visible content
          let loader = getDocsLoader(url, isDocusaurus);
          // Load the raw text from the web page
          const raw_documents = await loader.load();
          const title = (/<title>(.*)<\/title>/.exec(
            raw_documents[0].pageContent,
          ) ?? [])[1];
          console.log(`Title: ${title}`);

          // Convert the HTML to Markdown
          const md = turndownService.turndown(
            // pageContent always starts with added <title> tag, so this must be removed
            //  (there shouldn't be other <title> tags in there since it's the page body that's returned)
            raw_documents[0].pageContent.replace(/<title>(.*)<\/title>/, ""),
          );
          console.log("md", md);

          // Extract headings and subheadings from the Markdown
          const headerToSection = splitTextByHeaders(md);

          // For each heading, split the text into chunks by newline, filter out useless chunks, and split into 3s
          const doc_chunks = Object.entries(headerToSection)
            .map(([header, section]) => {
              // If a line is just the title, skip it
              if (!section.replaceAll(title, "").trim()) return [];
              // Split the documents into text chunks
              const lines = splitIntoTextChunks(section, [
                title,
                ...ignoreLines,
              ]);

              const chunkGroups = lines
                // Remove null values (at end of array) e.g. ["text", null, null] -> ["text"]
                .map((line, idx) =>
                  [line, lines[idx + 1], lines[idx + 2]].filter(
                    (item) => item !== null,
                  ),
                )
                // Slice to remove the last 2 chunk groups which are shorter than the rest
                // Max to ensure we still embed very short doc pages