(url, temp_file, proxies=None, resume_size=0, user_agent: Union[Dict, str, None] = None)
| 615 | |
| 616 | |
| 617 | def http_get(url, temp_file, proxies=None, resume_size=0, user_agent: Union[Dict, str, None] = None): |
| 618 | ua = "transformers/{}; python/{}".format(__version__, sys.version.split()[0]) |
| 619 | if is_torch_available(): |
| 620 | ua += "; torch/{}".format(torch.__version__) |
| 621 | if is_tf_available(): |
| 622 | ua += "; tensorflow/{}".format(tf.__version__) |
| 623 | if isinstance(user_agent, dict): |
| 624 | ua += "; " + "; ".join("{}/{}".format(k, v) for k, v in user_agent.items()) |
| 625 | elif isinstance(user_agent, str): |
| 626 | ua += "; " + user_agent |
| 627 | headers = {"user-agent": ua} |
| 628 | if resume_size > 0: |
| 629 | headers["Range"] = "bytes=%d-" % (resume_size,) |
| 630 | response = requests.get(url, stream=True, proxies=proxies, headers=headers) |
| 631 | if response.status_code == 416: # Range not satisfiable |
| 632 | return |
| 633 | content_length = response.headers.get("Content-Length") |
| 634 | total = resume_size + int(content_length) if content_length is not None else None |
| 635 | progress = tqdm( |
| 636 | unit="B", |
| 637 | unit_scale=True, |
| 638 | total=total, |
| 639 | initial=resume_size, |
| 640 | desc="Downloading", |
| 641 | disable=bool(logger.getEffectiveLevel() == logging.NOTSET), |
| 642 | ) |
| 643 | for chunk in response.iter_content(chunk_size=1024): |
| 644 | if chunk: # filter out keep-alive new chunks |
| 645 | progress.update(len(chunk)) |
| 646 | temp_file.write(chunk) |
| 647 | progress.close() |
| 648 | |
| 649 | |
| 650 | def get_from_cache( |
no test coverage detected