Download the contents of a remote url and return the text
(url: str, timeout: int=None)
| 163 | |
| 164 | @enforce_types |
| 165 | def download_url(url: str, timeout: int=None) -> str: |
| 166 | """Download the contents of a remote url and return the text""" |
| 167 | from .config import TIMEOUT, CHECK_SSL_VALIDITY, WGET_USER_AGENT |
| 168 | timeout = timeout or TIMEOUT |
| 169 | response = requests.get( |
| 170 | url, |
| 171 | headers={'User-Agent': WGET_USER_AGENT}, |
| 172 | verify=CHECK_SSL_VALIDITY, |
| 173 | timeout=timeout, |
| 174 | ) |
| 175 | |
| 176 | content_type = response.headers.get('Content-Type', '') |
| 177 | encoding = http_content_type_encoding(content_type) or html_body_declared_encoding(response.text) |
| 178 | |
| 179 | if encoding is not None: |
| 180 | response.encoding = encoding |
| 181 | |
| 182 | return response.text |
| 183 | |
| 184 | @enforce_types |
| 185 | def get_headers(url: str, timeout: int=None) -> str: |
no test coverage detected