Method crawlSingleDomainForCatalog

server/src/crawler.ts:1021–1060

(domain: string)

  }

  private async crawlSingleDomainForCatalog(domain: string): Promise<void> {
    const validation = await this.adAgentsManager.validateDomain(domain);
    if (!validation.valid || !validation.raw_data?.authorized_agents) return;

    await this.cacheAdagentsManifest(domain, validation.raw_data as AdagentsManifest);

    for (const authorizedAgent of validation.raw_data.authorized_agents) {
      if (!authorizedAgent.url) continue;

      await this.federatedIndex.recordAgentFromAdagentsJson(
        authorizedAgent.url,
        domain,
        authorizedAgent.authorized_for,
        authorizedAgent.property_ids
      );

      await this.recordPropertiesForAgent(
        validation.raw_data.properties || [],
        domain,
        authorizedAgent.url,
        authorizedAgent.authorized_for,
        authorizedAgent.property_ids
      );
    }

    if (this.eventsDb) {
      await this.eventsDb.writeEvent({
        event_type: 'publisher.adagents_discovered',
        entity_type: 'publisher',
        entity_id: domain,
        payload: {
          publisher_domain: domain,
          agent_count: validation.raw_data.authorized_agents.length,
          property_count: validation.raw_data.properties?.length ?? 0,
          source: 'catalog_crawl',
        },
        actor: 'pipeline:catalog_crawl',
      });
    }
  }

  private async processWithConcurrency<T, R>(
    items: T[],