HTTP client based on the `HTTPX` library. This client uses the `HTTPX` library to perform HTTP requests in crawlers (`BasicCrawler` subclasses) and to manage sessions, proxies, and error handling. See the `HttpClient` class for more common information about HTTP clients. ### Usage
| 85 | |
| 86 | @docs_group('HTTP clients') |
| 87 | class HttpxHttpClient(HttpClient): |
| 88 | """HTTP client based on the `HTTPX` library. |
| 89 | |
| 90 | This client uses the `HTTPX` library to perform HTTP requests in crawlers (`BasicCrawler` subclasses) |
| 91 | and to manage sessions, proxies, and error handling. |
| 92 | |
| 93 | See the `HttpClient` class for more common information about HTTP clients. |
| 94 | |
| 95 | ### Usage |
| 96 | |
| 97 | ```python |
| 98 | from crawlee.crawlers import HttpCrawler # or any other HTTP client-based crawler |
| 99 | from crawlee.http_clients import HttpxHttpClient |
| 100 | |
| 101 | http_client = HttpxHttpClient() |
| 102 | crawler = HttpCrawler(http_client=http_client) |
| 103 | ``` |
| 104 | """ |
| 105 | |
| 106 | _DEFAULT_HEADER_GENERATOR = HeaderGenerator() |
| 107 | |
| 108 | def __init__( |
| 109 | self, |
| 110 | *, |
| 111 | persist_cookies_per_session: bool = True, |
| 112 | http1: bool = True, |
| 113 | http2: bool = True, |
| 114 | verify: str | bool | SSLContext = True, |
| 115 | header_generator: HeaderGenerator | None = _DEFAULT_HEADER_GENERATOR, |
| 116 | **async_client_kwargs: Any, |
| 117 | ) -> None: |
| 118 | """Initialize a new instance. |
| 119 | |
| 120 | Args: |
| 121 | persist_cookies_per_session: Whether to persist cookies per HTTP session. |
| 122 | http1: Whether to enable HTTP/1.1 support. |
| 123 | http2: Whether to enable HTTP/2 support. |
| 124 | verify: SSL certificates used to verify the identity of requested hosts. |
| 125 | header_generator: Header generator instance to use for generating common headers. |
| 126 | async_client_kwargs: Additional keyword arguments for `httpx.AsyncClient`. |
| 127 | """ |
| 128 | super().__init__( |
| 129 | persist_cookies_per_session=persist_cookies_per_session, |
| 130 | ) |
| 131 | self._http1 = http1 |
| 132 | self._http2 = http2 |
| 133 | |
| 134 | self._async_client_kwargs = async_client_kwargs |
| 135 | self._header_generator = header_generator |
| 136 | |
| 137 | self._ssl_context = httpx.create_ssl_context(verify=verify) |
| 138 | |
| 139 | self._transport: _HttpxTransport | None = None |
| 140 | |
| 141 | self._client_by_proxy_url = dict[str | None, httpx.AsyncClient]() |
| 142 | |
| 143 | @override |
| 144 | async def crawl( |