Method from_content

src/crawlee/_utils/robots.py:38–46

Create a `RobotsTxtFile` instance from the given content. Args: url: The URL associated with the robots.txt file. content: The raw string content of the robots.txt file to be parsed.

(cls, url: str, content: str)

    @classmethod
    async def from_content(cls, url: str, content: str) -> Self:
        """Create a `RobotsTxtFile` instance from the given content.

        Args:
            url: The URL associated with the robots.txt file.
            content: The raw string content of the robots.txt file to be parsed.
        """
        robots = Protego.parse(content)
        return cls(url, robots)

    @classmethod
    async def load(cls, url: str, http_client: HttpClient, proxy_info: ProxyInfo | None = None) -> Self: