Method _get_client

src/crawlee/http_clients/_impit.py:225–254

Retrieve or create an HTTP client for the given proxy URL. If a client for the specified proxy URL does not exist, create and store a new one.

(self, proxy_url: str | None, cookie_jar: CookieJar | None)

            response.close()

    def _get_client(self, proxy_url: str | None, cookie_jar: CookieJar | None) -> AsyncClient:
        """Retrieve or create an HTTP client for the given proxy URL.

        If a client for the specified proxy URL does not exist, create and store a new one.
        """
        cached_data = self._client_by_proxy_url.get(proxy_url)
        if cached_data:
            client = cached_data['client']
            client_cookie_jar = cached_data['cookie_jar']
            if client_cookie_jar is cookie_jar:
                # If the cookie jar matches, return the existing client.
                return client

        # Prepare a default kwargs for the new client.
        kwargs: dict[str, Any] = {
            'proxy': proxy_url,
            'http3': self._http3,
            'verify': self._verify,
            'follow_redirects': True,
            'browser': self._browser,
        }

        # Update the default kwargs with any additional user-provided kwargs.
        kwargs.update(self._async_client_kwargs)

        client = AsyncClient(**kwargs, cookie_jar=cookie_jar)

        self._client_by_proxy_url[proxy_url] = _ClientCacheEntry(client=client, cookie_jar=cookie_jar)

        return client

    @staticmethod
    def _is_proxy_error(error: HTTPError) -> bool: