Method crawl

core/indexing/docs/crawlers/CheerioCrawler.ts:27–86

()

  ) {}

  async *crawl(): AsyncGenerator<PageData> {
    let currentPageCount = 0;
    console.log(
      `[${
        (this.constructor as any).name
      }] Starting crawl from: ${this.startUrl} - Max Depth: ${this.maxDepth}`,
    );

    const { baseUrl, basePath } = this.splitUrl(this.startUrl);

    let paths: { path: string; depth: number }[] = [
      { path: basePath, depth: 0 },
    ];

    let index = 0;

    while (index < paths.length) {
      const batch = paths.slice(index, index + 50);

      try {
        const promises = batch.map(({ path, depth }) =>
          this.getLinksFromUrl(baseUrl, path).then((links) => ({
            links,
            path,
            depth,
          })),
        );

        const results = await Promise.all(promises);
        for (const {
          links: { html, links: linksArray },
          path,
          depth,
        } of results) {
          if (html !== "" && depth <= this.maxDepth) {
            yield { url: this.startUrl.toString(), path, content: html };
            currentPageCount++;
            if (currentPageCount >= this.maxRequestsPerCrawl) {
              console.log("Crawl completed - max requests reached");
              return;
            }
          }

          if (depth < this.maxDepth) {
            for (let link of linksArray) {
              if (!paths.some((p) => p.path === link)) {
                paths.push({ path: link, depth: depth + 1 });
              }
            }
          }
        }
      } catch (e) {
        console.debug("Error while crawling page: ", e);
      }

      index += batch.length;
    }