Download the given URL and return a binary-mode file object to access the data.
(
url: str,
num_attempts: int = 10,
verbose: bool = True,
return_filename: bool = False,
)
| 76 | |
| 77 | |
| 78 | def open_url( |
| 79 | url: str, |
| 80 | num_attempts: int = 10, |
| 81 | verbose: bool = True, |
| 82 | return_filename: bool = False, |
| 83 | ) -> Any: |
| 84 | """Download the given URL and return a binary-mode file object to access the data.""" |
| 85 | assert num_attempts >= 1 |
| 86 | |
| 87 | # Doesn't look like an URL scheme so interpret it as a local filename. |
| 88 | if not re.match("^[a-z]+://", url): |
| 89 | return url if return_filename else open(url, "rb") |
| 90 | |
| 91 | # Handle file URLs. This code handles unusual file:// patterns that |
| 92 | # arise on Windows: |
| 93 | # |
| 94 | # file:///c:/foo.txt |
| 95 | # |
| 96 | # which would translate to a local '/c:/foo.txt' filename that's |
| 97 | # invalid. Drop the forward slash for such pathnames. |
| 98 | # |
| 99 | # If you touch this code path, you should test it on both Linux and |
| 100 | # Windows. |
| 101 | # |
| 102 | # Some internet resources suggest using urllib.request.url2pathname() but |
| 103 | # but that converts forward slashes to backslashes and this causes |
| 104 | # its own set of problems. |
| 105 | if url.startswith("file://"): |
| 106 | filename = urllib.parse.urlparse(url).path |
| 107 | if re.match(r"^/[a-zA-Z]:", filename): |
| 108 | filename = filename[1:] |
| 109 | return filename if return_filename else open(filename, "rb") |
| 110 | |
| 111 | url_md5 = hashlib.md5(url.encode("utf-8")).hexdigest() |
| 112 | |
| 113 | # Download. |
| 114 | url_name = None |
| 115 | url_data = None |
| 116 | with requests.Session() as session: |
| 117 | if verbose: |
| 118 | print("Downloading %s ..." % url, end="", flush=True) |
| 119 | for attempts_left in reversed(range(num_attempts)): |
| 120 | try: |
| 121 | with session.get(url) as res: |
| 122 | res.raise_for_status() |
| 123 | if len(res.content) == 0: |
| 124 | raise IOError("No data received") |
| 125 | |
| 126 | if len(res.content) < 8192: |
| 127 | content_str = res.content.decode("utf-8") |
| 128 | if "download_warning" in res.headers.get("Set-Cookie", ""): |
| 129 | links = [ |
| 130 | html.unescape(link) |
| 131 | for link in content_str.split('"') |
| 132 | if "export=download" in link |
| 133 | ] |
| 134 | if len(links) == 1: |
| 135 | url = requests.compat.urljoin(url, links[0]) |