Method _combine_headers

src/crawlee/http_clients/_httpx.py:325–337

Merge default headers with explicit headers for an HTTP request. Generate a final set of request headers by combining default headers, a random User-Agent header, and any explicitly provided headers.

(self, explicit_headers: HttpHeaders | None)

        return self._client_by_proxy_url[proxy_url]

    def _combine_headers(self, explicit_headers: HttpHeaders | None) -> HttpHeaders | None:
        """Merge default headers with explicit headers for an HTTP request.

        Generate a final set of request headers by combining default headers, a random User-Agent header,
        and any explicitly provided headers.
        """
        common_headers = self._header_generator.get_common_headers() if self._header_generator else HttpHeaders()
        user_agent_header = (
            self._header_generator.get_random_user_agent_header() if self._header_generator else HttpHeaders()
        )
        explicit_headers = explicit_headers or HttpHeaders()
        headers = common_headers | user_agent_header | explicit_headers
        return headers or None

    @staticmethod
    def _is_proxy_error(error: httpx.TransportError) -> bool: