Method crawl

packages/llm/src/langchain.ts:169–182

(url: string, options: CrawlOptions)

  @measure
  private static async crawl(url: string, options: CrawlOptions) {
    const { output, selectors } = options;
    const exists = await FileStore.has({ dir: output });
    if (exists) {
      return;
    }

    const { documents } = await WebCrawler.crawl({
      url,
      selectors,
    });

    await FileStore.write({ dir: output, files: documents });
  }

  @measure
  private static async storeFunction({