Function get_headers

archivebox/util.py:185–217

Download the contents of a remote url and return the headers

(url: str, timeout: int=None)

@enforce_types
def get_headers(url: str, timeout: int=None) -> str:
    """Download the contents of a remote url and return the headers"""
    from .config import TIMEOUT, CHECK_SSL_VALIDITY, WGET_USER_AGENT
    timeout = timeout or TIMEOUT

    try:
        response = requests.head(
            url,
            headers={'User-Agent': WGET_USER_AGENT},
            verify=CHECK_SSL_VALIDITY,
            timeout=timeout,
            allow_redirects=True,
        )
        if response.status_code >= 400:
            raise RequestException
    except ReadTimeout:
        raise
    except RequestException:
        response = requests.get(
            url,
            headers={'User-Agent': WGET_USER_AGENT},
            verify=CHECK_SSL_VALIDITY,
            timeout=timeout,
            stream=True
        )
    
    return pyjson.dumps(
        {
            'Status-Code': response.status_code,
            **dict(response.headers),
        },
        indent=4,
    )


@enforce_types