Method crawlCatalogDomains

server/src/crawler.ts:924–1019

()

  }

  async crawlCatalogDomains(): Promise<{ checked: number; found: number }> {
    if (this.catalogCrawling || this.crawling) {
      log.debug('Crawl already in progress, skipping catalog crawl');
      return { checked: 0, found: 0 };
    }

    this.catalogCrawling = true;
    const BATCH_SIZE = 100;
    const CONCURRENCY = 20;

    try {
      // Pull domains from queue, oldest requests first, respecting backoff
      const rows = await query<{ identifier_type: string; identifier_value: string }>(
        `SELECT identifier_type, identifier_value
         FROM catalog_crawl_queue
         WHERE identifier_type = 'domain'
           AND next_crawl_after <= NOW()
           AND found_adagents = FALSE
         ORDER BY crawl_requested_at ASC
         LIMIT $1`,
        [BATCH_SIZE]
      );

      if (rows.rows.length === 0) {
        this.catalogCrawling = false;
        return { checked: 0, found: 0 };
      }

      log.info({ count: rows.rows.length }, 'Catalog domain crawl batch');

      let found = 0;
      const domains = rows.rows.map(r => r.identifier_value);

      // Process with concurrency limit
      const results = await this.processWithConcurrency(
        domains,
        CONCURRENCY,
        async (domain) => {
          try {
            const validation = await this.adAgentsManager.validateDomain(domain);
            return { domain, valid: validation.valid && !!validation.raw_data?.authorized_agents };
          } catch {
            return { domain, valid: false };
          }
        }
      );

      for (const { domain, valid } of results) {
        if (valid) {
          found++;
          // Run full single-domain crawl to record agents/properties
          try {
            await this.crawlSingleDomainForCatalog(domain);
          } catch (err) {
            log.error({ domain, err }, 'Catalog domain crawl failed for domain');
          }

          // Mark as found in queue