(
url, temp_file, proxies=None, resume_size=0, headers=None, cookies=None, timeout=300.0, max_retries=3, desc=None
)
| 130 | |
| 131 | |
| 132 | def http_get_ms( |
| 133 | url, temp_file, proxies=None, resume_size=0, headers=None, cookies=None, timeout=300.0, max_retries=3, desc=None |
| 134 | ) -> Optional[requests.Response]: |
| 135 | logger.debug( |
| 136 | '[MS_DOWNLOAD] http_get_ms entry: url=%s, timeout=%s, resume_size=%s', |
| 137 | url, timeout, resume_size, |
| 138 | ) |
| 139 | headers = dict(headers) if headers is not None else {} |
| 140 | headers['user-agent'] = get_datasets_user_agent_ms(user_agent=headers.get('user-agent')) |
| 141 | if resume_size > 0: |
| 142 | headers['Range'] = f'bytes={resume_size:d}-' |
| 143 | response = _request_with_retry_ms( |
| 144 | method='GET', |
| 145 | url=url, |
| 146 | stream=True, |
| 147 | proxies=proxies, |
| 148 | headers=headers, |
| 149 | cookies=cookies, |
| 150 | max_retries=max_retries, |
| 151 | timeout=timeout, |
| 152 | ) |
| 153 | if temp_file is None: |
| 154 | return response |
| 155 | if response.status_code == 416: # Range not satisfiable |
| 156 | return |
| 157 | content_length = response.headers.get('Content-Length') |
| 158 | total = resume_size + int(content_length) if content_length is not None else None |
| 159 | |
| 160 | progress = tqdm(total=total, initial=resume_size, unit_scale=True, unit='B', desc=desc or 'Downloading') |
| 161 | for chunk in response.iter_content(chunk_size=1024): |
| 162 | progress.update(len(chunk)) |
| 163 | temp_file.write(chunk) |
| 164 | |
| 165 | progress.close() |
| 166 | |
| 167 | |
| 168 | def get_from_cache_ms( |
no test coverage detected
searching dependent graphs…