Load the robots.txt file for a given URL. Args: url: The direct URL of the robots.txt file to be loaded. http_client: The `HttpClient` instance used to perform the network request for fetching the robots.txt file. proxy_info: Optional `ProxyInfo` to be us
(cls, url: str, http_client: HttpClient, proxy_info: ProxyInfo | None = None)
| 47 | |
| 48 | @classmethod |
| 49 | async def load(cls, url: str, http_client: HttpClient, proxy_info: ProxyInfo | None = None) -> Self: |
| 50 | """Load the robots.txt file for a given URL. |
| 51 | |
| 52 | Args: |
| 53 | url: The direct URL of the robots.txt file to be loaded. |
| 54 | http_client: The `HttpClient` instance used to perform the network request for fetching the robots.txt file. |
| 55 | proxy_info: Optional `ProxyInfo` to be used when fetching the robots.txt file. If None, no proxy is used. |
| 56 | """ |
| 57 | try: |
| 58 | response = await http_client.send_request(url, proxy_info=proxy_info) |
| 59 | |
| 60 | body = ( |
| 61 | b'User-agent: *\nAllow: /' |
| 62 | if is_status_code_client_error(response.status_code) |
| 63 | else await response.read() |
| 64 | ) |
| 65 | robots = Protego.parse(body.decode('utf-8')) |
| 66 | |
| 67 | except Exception as e: |
| 68 | logger.warning(f'Failed to fetch from robots.txt from "{url}" with error: "{e}"') |
| 69 | |
| 70 | robots = Protego.parse('User-agent: *\nAllow: /') |
| 71 | |
| 72 | return cls(url, robots, http_client=http_client, proxy_info=proxy_info) |
| 73 | |
| 74 | @classmethod |
| 75 | async def find(cls, url: str, http_client: HttpClient, proxy_info: ProxyInfo | None = None) -> Self: |
no test coverage detected