Method crawlToStore

packages/llm/src/langchain.ts:99–115

({
    url,
    communityName,
    options,
  }: {
    url: string;
    communityName: string;
    options: CrawlOptions;
  })

  }

  static async crawlToStore({
    url,
    communityName,
    options,
  }: {
    url: string;
    communityName: string;
    options: CrawlOptions;
  }) {
    await this.crawl(url, options);
    const docs = await this.getDocuments(url);
    const splitDocs = await this.splitDocuments(docs);
    const vectorStore = await this.storeFunction({ splitDocs });
    const directory = `.db/${communityName}`;
    fs.mkdirSync(directory, { recursive: true });
    await vectorStore.save(directory);
  }

  @measure
  private static async summarize(text: string) {