Function getDocsLoader

lib/embed-docs/embedDocs.ts:123–223

(
  url: string,
  isDocusaurus: boolean,
)

}

export function getDocsLoader(
  url: string,
  isDocusaurus: boolean,
): PlaywrightWebBaseLoader {
  return new PlaywrightWebBaseLoader(url, {
    launchOptions: { headless: true },
    gotoOptions: { waitUntil: "domcontentloaded" },
    evaluate: isDocusaurus
      ? async function (page: Page, browser: Browser) {
          await page.waitForLoadState("domcontentloaded");
          await new Promise((resolve) => setTimeout(resolve, 10000));
          let out = "";
          while (!out) {
            await new Promise((resolve) => setTimeout(resolve, 1000));
            console.log("Attempting to get HTML");
            out = await page.evaluate(() => {
              function removeFromHtml(node: HTMLBodyElement, tags: string[]) {
                for (const tag of tags) {
                  let elements = node.getElementsByTagName(tag);
                  while (elements.length) {
                    if (!elements[0]) break;
                    elements[0].parentNode?.removeChild(elements[0]);
                    elements = node.getElementsByTagName(tag);
                  }
                }
                return node;
              }

              let clone = document.body.cloneNode(true) as HTMLBodyElement;
              if (!clone) return "";
              const mains = clone.getElementsByTagName("main");
              const main = mains[mains.length - 1];
              // @ts-ignore
              if (main) clone = main;
              // In Docusaurus, the title is <page name> | <site name>
              const title = document.title.split("|")[0];

              clone = removeFromHtml(clone, [
                "script",
                "aside",
                "button",
                "header",
                "nav",
                "footer",
                "style",
                "noscript",
              ]);
              // In Docusaurus, the table of contents can be picked up if we don't smite it
              clone.getElementsByClassName("table-of-contents")[0]?.remove();

              return `<title>${title}</title>${clone.innerHTML}`;
            });
          }
          return out;
        }
      : async function (page: Page, browser: Browser) {
          await page.waitForLoadState("domcontentloaded");
          await new Promise((resolve) => setTimeout(resolve, 10000));