GET ``url`` and return the body as ``Fetched``. Streams the response so we can short-circuit on payloads larger than ``max_bytes`` without buffering the whole thing into memory. Args: url: Absolute http(s) URL. timeout: Per-request timeout in seconds. max_bytes:
(
url: str,
*,
timeout: float = 30.0,
max_bytes: int = 50_000_000,
user_agent: str = DEFAULT_USER_AGENT,
)
| 24 | "last-modified", |
| 25 | "content-disposition", |
| 26 | "retry-after", |
| 27 | ) |
| 28 | |
| 29 | |
| 30 | @dataclass(frozen=True, slots=True) |
| 31 | class FetchPolicy: |
| 32 | """Generic retry and per-host politeness settings for HTTP requests.""" |
| 33 | |
| 34 | max_attempts: int = 3 |
| 35 | backoff_base_seconds: float = 0.5 |
| 36 | backoff_max_seconds: float = 30.0 |
| 37 | jitter_seconds: float = 0.1 |
| 38 | max_concurrency_per_host: int = 2 |
| 39 | min_interval_seconds: float = 0.0 |
| 40 | |
| 41 | def __post_init__(self) -> None: |
| 42 | """Validate policy values before requests start.""" |
| 43 | if self.max_attempts < 1: |
| 44 | raise ValueError("max_attempts must be >= 1") |
| 45 | if self.backoff_base_seconds < 0: |
| 46 | raise ValueError("backoff_base_seconds must be >= 0") |
| 47 | if self.backoff_max_seconds < self.backoff_base_seconds: |
| 48 | raise ValueError( |
| 49 | "backoff_max_seconds must be >= backoff_base_seconds" |
| 50 | ) |
| 51 | if self.jitter_seconds < 0: |
| 52 | raise ValueError("jitter_seconds must be >= 0") |
| 53 | if self.max_concurrency_per_host < 1: |
| 54 | raise ValueError("max_concurrency_per_host must be >= 1") |
| 55 | if self.min_interval_seconds < 0: |
| 56 | raise ValueError("min_interval_seconds must be >= 0") |
| 57 | |
| 58 | |
| 59 | class FetchAttemptsExhausted(httpx.HTTPError): |
| 60 | """A retryable HTTP failure that exhausted its configured attempts.""" |
| 61 | |
| 62 | def __init__( |
| 63 | self, |
| 64 | *, |
| 65 | url: str, |
| 66 | attempts: int, |
| 67 | last_error: httpx.HTTPError, |
| 68 | ) -> None: |
| 69 | self.url = url |
| 70 | self.attempts = attempts |
| 71 | self.last_error = last_error |
| 72 | super().__init__( |
| 73 | f"request to {url!r} failed after {attempts} attempts: {last_error}" |
| 74 | ) |
| 75 | |
| 76 | |
| 77 | @dataclass(slots=True) |
| 78 | class _HostState: |
| 79 | semaphore: asyncio.Semaphore |
| 80 | spacing_lock: asyncio.Lock |
| 81 | next_request_at: float = 0.0 |
| 82 | |
| 83 |