Function getNestedUrls

lib/embed-docs/getNestedUrls.ts:11–114

(
  url_docs: string,
  recursive_depth: number = 1,
  current_depth: number = 1,
  base_url: string = "",
)

}

export async function getNestedUrls(
  url_docs: string,
  recursive_depth: number = 1,
  current_depth: number = 1,
  base_url: string = "",
): Promise<Link[]> {
  /**
    Get all links from a web page up to a specified recursion depth.

    url_docs: the URL of the web page
    recursive_depth: the maximum recursion depth
    current_depth: the current recursion depth
    **/
  // TODO: Deal with # links to specific elements on a page

  // Check if we have reached the maximum recursion depth
  if (current_depth > recursive_depth && recursive_depth !== 0) return [];
  else if (recursive_depth == 0) return [{ name: "", href: url_docs }];
  const links: Link[] = [];
  base_url = base_url || url_docs;
  try {
    console.log("Getting links from " + url_docs);

    const loader = new CheerioWebBaseLoader(url_docs);
    const page = await loader.scrape();
    page("a").each((index, element) => {
      const link = page(element);
      if (link.attr("href")) {
        links.push({
          name: link.text(),
          href: makeUrlAbsolute(url_docs, link.attr("href")!),
        });
      } else {
      }
    });
  } catch (e) {
    console.log(e);
  }

  // Try using Playwright if Cheerio doesn't come up with the goods
  if (links.length < 5) {
    console.log("Trying playwright");
    const loader = new PlaywrightWebBaseLoader(url_docs, {
      launchOptions: {
        headless: true,
      },
      gotoOptions: { waitUntil: "domcontentloaded" },
      async evaluate(page: Page, browser: Browser) {
        await page.waitForLoadState("domcontentloaded");
        await new Promise((resolve) => setTimeout(resolve, 5000));
        return await page.evaluate(() => {
          const out = [];
          const links = document.getElementsByTagName("a");
          for (const link of links) {
            out.push({
              name: link.text,
              href: link.href,
            });