Method is_allowed

src/crawlee/_utils/robots.py:86–96

Check if the given URL is allowed for the given user agent. Args: url: The URL to check against the robots.txt rules. user_agent: The user-agent string to check permissions for. Defaults to '*' which matches any user-agent.

(self, url: str, user_agent: str = '*')

        return await cls.load(str(robots_url), http_client, proxy_info)

    def is_allowed(self, url: str, user_agent: str = '*') -> bool:
        """Check if the given URL is allowed for the given user agent.

        Args:
            url: The URL to check against the robots.txt rules.
            user_agent: The user-agent string to check permissions for. Defaults to '*' which matches any user-agent.
        """
        check_url = URL(url)
        if check_url.origin() != self._original_url:
            return True
        return bool(self._robots.can_fetch(str(check_url), user_agent))

    def get_sitemaps(self, *, enqueue_strategy: EnqueueStrategy) -> list[str]:
        """Get the list of sitemap URLs from the robots.txt file, filtered by enqueue strategy.