Method crawlSingleDomain

server/src/crawler.ts:840–908

(domain: string)

  // ── Single Domain Crawl ──────────────────────────────────────────

  async crawlSingleDomain(domain: string): Promise<void> {
    if (this.crawling) {
      log.warn({ domain }, 'Full crawl in progress, skipping single domain crawl');
      return;
    }

    log.info({ domain }, 'Single domain crawl requested');

    try {
      const validation = await this.adAgentsManager.validateDomain(domain);

      if (!validation.valid || !validation.raw_data?.authorized_agents) {
        log.warn({ domain }, 'No valid adagents.json for domain');
        return;
      }

      await this.cacheAdagentsManifest(domain, validation.raw_data as AdagentsManifest);

      // Record agents and properties
      for (const authorizedAgent of validation.raw_data.authorized_agents) {
        if (!authorizedAgent.url) continue;

        await this.federatedIndex.recordAgentFromAdagentsJson(
          authorizedAgent.url,
          domain,
          authorizedAgent.authorized_for,
          authorizedAgent.property_ids
        );

        await this.recordPropertiesForAgent(
          validation.raw_data.properties || [],
          domain,
          authorizedAgent.url,
          authorizedAgent.authorized_for,
          authorizedAgent.property_ids
        );
      }

      // Produce per-domain events
      if (this.eventsDb) {
        await this.eventsDb.writeEvent({
          event_type: 'publisher.adagents_changed',
          entity_type: 'publisher',
          entity_id: domain,
          payload: {
            publisher_domain: domain,
            agent_count: validation.raw_data.authorized_agents.length,
            property_count: validation.raw_data.properties?.length ?? 0,
          },
          actor: 'api:crawl-request',
        });
      }

      // Scan brand.json for this domain
      try {
        await this.scanBrandForDomain(domain);
      } catch (err) {
        log.warn({ domain, err: err instanceof Error ? err.message : err }, 'Brand scan during single domain crawl failed');