Method get_sitemaps

src/crawlee/_utils/robots.py:98–116

Get the list of sitemap URLs from the robots.txt file, filtered by enqueue strategy. Args: enqueue_strategy: Strategy used to filter sitemap entries relative to the robots.txt URL's host. Pass `'same-hostname'` to match the sitemap protocol's same-host expectatio

(self, *, enqueue_strategy: EnqueueStrategy)

        return bool(self._robots.can_fetch(str(check_url), user_agent))

    def get_sitemaps(self, *, enqueue_strategy: EnqueueStrategy) -> list[str]:
        """Get the list of sitemap URLs from the robots.txt file, filtered by enqueue strategy.

        Args:
            enqueue_strategy: Strategy used to filter sitemap entries relative to the robots.txt URL's host.
                Pass `'same-hostname'` to match the sitemap protocol's same-host expectation, or `'all'` to
                disable host filtering. Regardless of the strategy, entries with non-`http(s)` schemes are
                always filtered out.
        """
        sitemaps: list[str] = []
        for sitemap_url in self._robots.sitemaps:
            ok, reason = filter_url(target=sitemap_url, strategy=enqueue_strategy, origin=self._original_url)
            if not ok:
                logger.warning(
                    f'Skipping sitemap {sitemap_url!r} listed in robots.txt at {str(self._original_url)!r}: {reason}.'
                )
                continue
            sitemaps.append(sitemap_url)
        return sitemaps

    def get_crawl_delay(self, user_agent: str = '*') -> int | None:
        """Get the crawl delay for the given user agent.