Method from_url

src/crawlee/_request.py:244–343

Create a new `Request` instance from a URL. This is recommended constructor for creating new `Request` instances. It generates a `Request` object from a given URL with additional options to customize HTTP method, payload, unique key, and other request properties. If no `uniq

(
        cls,
        url: str,
        *,
        method: HttpMethod = 'GET',
        headers: HttpHeaders | dict[str, str] | None = None,
        payload: HttpPayload | str | None = None,
        label: str | None = None,
        session_id: str | None = None,
        unique_key: str | None = None,
        keep_url_fragment: bool = False,
        use_extended_unique_key: bool = False,
        always_enqueue: bool = False,
        enqueue_strategy: EnqueueStrategy | None = None,
        max_retries: int | None = None,
        **kwargs: Any,
    )

    @classmethod
    def from_url(
        cls,
        url: str,
        *,
        method: HttpMethod = 'GET',
        headers: HttpHeaders | dict[str, str] | None = None,
        payload: HttpPayload | str | None = None,
        label: str | None = None,
        session_id: str | None = None,
        unique_key: str | None = None,
        keep_url_fragment: bool = False,
        use_extended_unique_key: bool = False,
        always_enqueue: bool = False,
        enqueue_strategy: EnqueueStrategy | None = None,
        max_retries: int | None = None,
        **kwargs: Any,
    ) -> Self:
        """Create a new `Request` instance from a URL.

        This is recommended constructor for creating new `Request` instances. It generates a `Request` object from
        a given URL with additional options to customize HTTP method, payload, unique key, and other request
        properties. If no `unique_key` or `id` is provided, they are computed automatically based on the URL,
        method and payload. It depends on the `keep_url_fragment` and `use_extended_unique_key` flags.

        Args:
            url: The URL of the request.
            method: The HTTP method of the request.
            headers: The HTTP headers of the request.
            payload: The data to be sent as the request body. Typically used with 'POST' or 'PUT' requests.
            label: A custom label to differentiate between request types. This is stored in `user_data`, and it is
                used for request routing (different requests go to different handlers).
            session_id: ID of a specific `Session` to which the request will be strictly bound.
                If the session becomes unavailable when the request is processed, a `RequestCollisionError` will be
                raised.
            unique_key: A unique key identifying the request. If not provided, it is automatically computed based on
                the URL and other parameters. Requests with the same `unique_key` are treated as identical.
            keep_url_fragment: Determines whether the URL fragment (e.g., `#section`) should be included in
                the `unique_key` computation. This is only relevant when `unique_key` is not provided.
            use_extended_unique_key: Determines whether to include the HTTP method, ID Session and payload in the
                `unique_key` computation. This is only relevant when `unique_key` is not provided.
            always_enqueue: If set to `True`, the request will be enqueued even if it is already present in the queue.
                Using this is not allowed when a custom `unique_key` is also provided and will result in a `ValueError`.
            enqueue_strategy: The strategy that will be used for enqueuing the request.
            max_retries: Maximum number of retries for this request. Allows to override the global `max_request_retries`
                option of `BasicCrawler`.
            **kwargs: Additional request properties.
        """
        if unique_key is not None and always_enqueue:
            raise ValueError('`always_enqueue` cannot be used with a custom `unique_key`')

        if isinstance(headers, dict) or headers is None:
            headers = HttpHeaders(headers or {})

        if isinstance(payload, str):
            payload = payload.encode()

        unique_key = unique_key or compute_unique_key(
            url,