MCPcopy Create free account
hub / github.com/apify/crawlee-python / add_request

Method add_request

src/crawlee/storages/_request_queue.py:178–198  ·  view source on GitHub ↗
(
        self,
        request: str | Request,
        *,
        forefront: bool = False,
    )

Source from the content-addressed store, hash-verified

176
177 @override
178 async def add_request(
179 self,
180 request: str | Request,
181 *,
182 forefront: bool = False,
183 ) -> ProcessedRequest | None:
184 request = self._transform_request(request)
185 # Route through `_process_batch` so a single add retries unprocessed requests just like a batched one.
186 response = await self._process_batch([request], base_retry_wait=timedelta(seconds=1), forefront=forefront)
187
188 if response.processed_requests:
189 return response.processed_requests[0]
190
191 # `_process_batch` already warns about requests left unprocessed after retries; only an empty response
192 # (neither processed nor unprocessed) is unexpected here.
193 if not response.unprocessed_requests:
194 logger.warning(
195 f'Request {request.url} was not processed by storage client '
196 f'"{self._client.__class__.__name__}" (received an empty response).'
197 )
198 return None
199
200 @override
201 async def add_requests(

Calls 2

_process_batchMethod · 0.95
_transform_requestMethod · 0.80