Function fetch_public_http_resource

helpers/network.py:99–173

(
    url: str,
    *,
    max_bytes: int,
    max_redirects: int = 5,
    timeout: tuple[float, float] = DEFAULT_FETCH_TIMEOUT,
)

def fetch_public_http_resource(
    url: str,
    *,
    max_bytes: int,
    max_redirects: int = 5,
    timeout: tuple[float, float] = DEFAULT_FETCH_TIMEOUT,
) -> HttpFetchResult:
    current_url = url
    session = requests.Session()
    session.trust_env = False

    for redirect_count in range(max_redirects + 1):
        validate_public_http_url(current_url)

        try:
            with session.get(
                current_url,
                stream=True,
                allow_redirects=False,
                headers=_build_request_headers(),
                timeout=timeout,
            ) as response:
                if 300 <= response.status_code < 400:
                    location = response.headers.get("Location")
                    if not location:
                        raise ValueError(
                            f"Remote URL redirect is missing a Location header: {current_url}"
                        )
                    if redirect_count >= max_redirects:
                        raise ValueError(
                            f"Remote URL exceeded redirect limit ({max_redirects}): {url}"
                        )
                    current_url = urljoin(current_url, location)
                    continue

                if response.status_code >= 400:
                    raise ValueError(
                        f"Remote URL returned HTTP {response.status_code}: {current_url}"
                    )

                content_length = response.headers.get("Content-Length")
                if content_length:
                    try:
                        declared_length = int(content_length)
                    except ValueError:
                        declared_length = None
                    if declared_length is not None and declared_length > max_bytes:
                        raise ValueError(
                            f"Remote document exceeds max size {max_bytes} bytes: {current_url}"
                        )

                body = bytearray()
                for chunk in response.iter_content(chunk_size=64 * 1024):
                    if not chunk:
                        continue
                    body.extend(chunk)
                    if len(body) > max_bytes:
                        raise ValueError(