Method crawlAllAgents

server/src/crawler.ts:54–130

(agents: Agent[])

  }

  async crawlAllAgents(agents: Agent[]): Promise<CrawlResult> {
    if (this.crawling) {
      log.debug('Crawl already in progress, skipping');
      return this.lastResult || this.emptyResult();
    }

    this.crawling = true;

    // Filter out agents whose owners have paused monitoring
    const pausedUrls = await this.getPausedAgentUrls();
    const activeAgents = agents.filter(a => !pausedUrls.has(a.url));
    if (activeAgents.length < agents.length) {
      log.info({ paused: agents.length - activeAgents.length, active: activeAgents.length }, 'Skipping paused agents');
    }
    log.info({ agentCount: activeAgents.length }, 'Starting crawl');

    // Convert our Agent type to AgentInfo for the crawler
    const agentInfos: AgentInfo[] = activeAgents.map((agent) => ({
      agent_url: agent.url,
      protocol: agent.protocol || "mcp", // Use agent's protocol, default to MCP
      publisher_domain: this.extractDomain(agent.url),
    }));

    try {
      const result = await this.crawler.crawlAgents(agentInfos);

      this.lastCrawl = new Date();
      this.lastResult = result;
      this.crawling = false;

      log.info({
        totalProperties: result.totalProperties,
        successfulAgents: result.successfulAgents,
        totalAgents: agents.length,
        publisherDomains: result.totalPublisherDomains,
      }, 'Crawl complete');

      if (result.failedAgents > 0) {
        log.warn({ failedAgents: result.failedAgents }, 'Some agents failed (no adagents.json)');
      }

      if (result.errors.length > 0) {
        const errorsByDomain: Record<string, string[]> = {};
        for (const err of result.errors) {
          (errorsByDomain[err.agent_url] ??= []).push(err.error);
        }
        log.warn({ errorsByDomain }, 'Crawl errors');
      }

      if (result.warnings && result.warnings.length > 0) {
        log.warn({ warnings: result.warnings }, 'Crawl warnings');
      }

      // Snapshot pre-crawl state for diffing
      const preCrawlAgents = await this.snapshotAgentState();

      // Populate federated index from PropertyIndex and adagents.json files
      const crawledDomains = await this.populateFederatedIndex(agents);