Method stream

src/crawlee/http_clients/_base.py:160–190

Stream an HTTP request via the client. This method should be used for downloading potentially large data where you need to process the response body in chunks rather than loading it entirely into memory. Args: url: The URL to send the request to. met

(
        self,
        url: str,
        *,
        method: HttpMethod = 'GET',
        headers: HttpHeaders | dict[str, str] | None = None,
        payload: HttpPayload | None = None,
        session: Session | None = None,
        proxy_info: ProxyInfo | None = None,
        timeout: timedelta | None = None,
    )

    @abstractmethod
    def stream(
        self,
        url: str,
        *,
        method: HttpMethod = 'GET',
        headers: HttpHeaders | dict[str, str] | None = None,
        payload: HttpPayload | None = None,
        session: Session | None = None,
        proxy_info: ProxyInfo | None = None,
        timeout: timedelta | None = None,
    ) -> AbstractAsyncContextManager[HttpResponse]:
        """Stream an HTTP request via the client.

        This method should be used for downloading potentially large data where you need to process
        the response body in chunks rather than loading it entirely into memory.

        Args:
            url: The URL to send the request to.
            method: The HTTP method to use.
            headers: The headers to include in the request.
            payload: The data to be sent as the request body.
            session: The session associated with the request.
            proxy_info: The information about the proxy to be used.
            timeout: The maximum time to wait for establishing the connection.

        Raises:
            ProxyError: Raised if a proxy-related error occurs.

        Returns:
            An async context manager yielding the HTTP response with streaming capabilities.
        """

    @abstractmethod
    async def cleanup(self) -> None: