Method indexAndAdd

core/indexing/docs/DocsService.ts:435–739

(
    siteIndexingConfig: SiteIndexingConfig,
    forceReindex: boolean = false,
  )

  // eslint-disable-next-line max-statements
  async indexAndAdd(
    siteIndexingConfig: SiteIndexingConfig,
    forceReindex: boolean = false,
  ): Promise<void> {
    const { startUrl, useLocalCrawling, maxDepth, faviconUrl } =
      siteIndexingConfig;

    // First, if indexing is already in process, don't attempt
    // This queue is necessary because indexAndAdd is invoked circularly by config edits
    // TODO shouldn't really be a gap between adding and checking in queue but probably fine
    if (this.docsIndexingQueue.has(startUrl)) {
      return;
    }

    const { provider } = await this.getEmbeddingsProvider();
    if (!provider) {
      console.warn("@docs indexAndAdd: no embeddings provider found");
      return;
    }

    const startedWithEmbedder = provider.embeddingId;

    // Check if doc has been successfully indexed with the given embedder
    const indexExists = await this.hasMetadata(startUrl);

    // Build status update - most of it is fixed values
    const fixedStatus: Omit<
      IndexingStatus,
      "progress" | "description" | "status"
    > = {
      type: "docs",
      id: siteIndexingConfig.startUrl,
      embeddingsProviderId: provider.embeddingId,
      isReindexing: forceReindex && indexExists,
      title: siteIndexingConfig.title,
      debugInfo: `max depth: ${siteIndexingConfig.maxDepth ?? "unlimited"}`,
      icon: siteIndexingConfig.faviconUrl,
      url: siteIndexingConfig.startUrl,
    };

    // If not force-reindexing and has failed with same config, don't reattempt
    if (!forceReindex) {
      if (hasIndexingFailed(siteIndexingConfig)) {
        console.log(
          `Not reattempting to index ${siteIndexingConfig.startUrl}, has already failed with same config`,
        );
        this.handleStatusUpdate({
          ...fixedStatus,
          description: "Failed",
          status: "failed",
          progress: 1,
        });
        return;
      }
    }

    if (indexExists && !forceReindex) {
      this.handleStatusUpdate({