Get contents of a URL and save to a file. https://github.com/huggingface/transformers/blob/master/src/transformers/file_utils.py
(url, out_file, proxies=None)
| 118 | |
| 119 | |
| 120 | def http_get(url, out_file, proxies=None): |
| 121 | """Get contents of a URL and save to a file. |
| 122 | |
| 123 | https://github.com/huggingface/transformers/blob/master/src/transformers/file_utils.py |
| 124 | """ |
| 125 | logger.info(f"Downloading {url}.") |
| 126 | req = requests.get(url, stream=True, proxies=proxies) |
| 127 | content_length = req.headers.get("Content-Length") |
| 128 | total = int(content_length) if content_length is not None else None |
| 129 | if req.status_code == 403 or req.status_code == 404: |
| 130 | raise Exception(f"Could not reach {url}.") |
| 131 | progress = tqdm.tqdm(unit="B", unit_scale=True, total=total) |
| 132 | for chunk in req.iter_content(chunk_size=1024): |
| 133 | if chunk: # filter out keep-alive new chunks |
| 134 | progress.update(len(chunk)) |
| 135 | out_file.write(chunk) |
| 136 | progress.close() |
| 137 | |
| 138 | |
| 139 | if sys.stdout.isatty(): |
no test coverage detected