Method parse_sitemaps

src/crawlee/_utils/robots.py:128–140

Parse the sitemaps from the robots.txt file and return a `Sitemap` instance. Args: enqueue_strategy: Forwarded to `get_sitemaps`; see that method for details.

(self, *, enqueue_strategy: EnqueueStrategy)

        return int(crawl_delay) if crawl_delay is not None else None

    async def parse_sitemaps(self, *, enqueue_strategy: EnqueueStrategy) -> Sitemap:
        """Parse the sitemaps from the robots.txt file and return a `Sitemap` instance.

        Args:
            enqueue_strategy: Forwarded to `get_sitemaps`; see that method for details.
        """
        sitemaps = self.get_sitemaps(enqueue_strategy=enqueue_strategy)
        if not self._http_client:
            raise ValueError('HTTP client is required to parse sitemaps.')

        return await Sitemap.load(
            sitemaps, self._http_client, self._proxy_info, parse_sitemap_options={'enqueue_strategy': enqueue_strategy}
        )

    async def parse_urls_from_sitemaps(self, *, enqueue_strategy: EnqueueStrategy) -> list[str]:
        """Parse the sitemaps in the robots.txt file and return a list URLs.