Download the given URL and return a binary-mode file object to access the data.
(url: str, cache_dir: str = None, num_attempts: int = 10, verbose: bool = True, return_filename: bool = False, cache: bool = True)
| 394 | |
| 395 | |
| 396 | def open_url(url: str, cache_dir: str = None, num_attempts: int = 10, verbose: bool = True, return_filename: bool = False, cache: bool = True) -> Any: |
| 397 | """Download the given URL and return a binary-mode file object to access the data.""" |
| 398 | assert num_attempts >= 1 |
| 399 | assert not (return_filename and (not cache)) |
| 400 | |
| 401 | # Doesn't look like an URL scheme so interpret it as a local filename. |
| 402 | if not re.match('^[a-z]+://', url): |
| 403 | return url if return_filename else open(url, "rb") |
| 404 | |
| 405 | # Handle file URLs. This code handles unusual file:// patterns that |
| 406 | # arise on Windows: |
| 407 | # |
| 408 | # file:///c:/foo.txt |
| 409 | # |
| 410 | # which would translate to a local '/c:/foo.txt' filename that's |
| 411 | # invalid. Drop the forward slash for such pathnames. |
| 412 | # |
| 413 | # If you touch this code path, you should test it on both Linux and |
| 414 | # Windows. |
| 415 | # |
| 416 | # Some internet resources suggest using urllib.request.url2pathname() but |
| 417 | # but that converts forward slashes to backslashes and this causes |
| 418 | # its own set of problems. |
| 419 | if url.startswith('file://'): |
| 420 | filename = urllib.parse.urlparse(url).path |
| 421 | if re.match(r'^/[a-zA-Z]:', filename): |
| 422 | filename = filename[1:] |
| 423 | return filename if return_filename else open(filename, "rb") |
| 424 | |
| 425 | assert is_url(url) |
| 426 | |
| 427 | # Lookup from cache. |
| 428 | if cache_dir is None: |
| 429 | cache_dir = make_cache_dir_path('downloads') |
| 430 | |
| 431 | url_md5 = hashlib.md5(url.encode("utf-8")).hexdigest() |
| 432 | if cache: |
| 433 | cache_files = glob.glob(os.path.join(cache_dir, url_md5 + "_*")) |
| 434 | if len(cache_files) == 1: |
| 435 | filename = cache_files[0] |
| 436 | return filename if return_filename else open(filename, "rb") |
| 437 | |
| 438 | # Download. |
| 439 | url_name = None |
| 440 | url_data = None |
| 441 | with requests.Session() as session: |
| 442 | if verbose: |
| 443 | print("Downloading %s ..." % url, end="", flush=True) |
| 444 | for attempts_left in reversed(range(num_attempts)): |
| 445 | try: |
| 446 | with session.get(url) as res: |
| 447 | res.raise_for_status() |
| 448 | if len(res.content) == 0: |
| 449 | raise IOError("No data received") |
| 450 | |
| 451 | if len(res.content) < 8192: |
| 452 | content_str = res.content.decode("utf-8") |
| 453 | if "download_warning" in res.headers.get("Set-Cookie", ""): |
nothing calls this directly
no test coverage detected