Download the data files specified in YAML file to a directory. Return false if the downloaded file or the local copy (if not overwrite) has a different checksum.
(data_dir, yaml_path, overwrite=False)
| 77 | |
| 78 | |
| 79 | def download(data_dir, yaml_path, overwrite=False): |
| 80 | """Download the data files specified in YAML file to a directory. |
| 81 | |
| 82 | Return false if the downloaded file or the local copy (if not overwrite) has a different checksum. |
| 83 | """ |
| 84 | sample_data = _loadYAML(yaml_path) |
| 85 | logger.info("Downloading data for %s", sample_data.sample) |
| 86 | |
| 87 | def _downloadFile(path, url): |
| 88 | logger.info("Downloading %s from %s", path, url) |
| 89 | import requests |
| 90 | |
| 91 | r = requests.get(url, stream=True, timeout=5) |
| 92 | size = int(r.headers.get("content-length", 0)) |
| 93 | from tqdm import tqdm |
| 94 | |
| 95 | progress_bar = tqdm(total=size, unit="iB", unit_scale=True) |
| 96 | with open(path, "wb") as fd: |
| 97 | for chunk in r.iter_content(chunk_size=1024): |
| 98 | progress_bar.update(len(chunk)) |
| 99 | fd.write(chunk) |
| 100 | progress_bar.close() |
| 101 | |
| 102 | allGood = True |
| 103 | for f in sample_data.files: |
| 104 | fpath = os.path.join(data_dir, f.path) |
| 105 | if os.path.exists(fpath): |
| 106 | if _checkMD5(fpath, f.checksum): |
| 107 | logger.info("Found local copy %s, skip downloading.", fpath) |
| 108 | continue |
| 109 | else: |
| 110 | logger.warning("Local copy %s has a different checksum!", fpath) |
| 111 | if overwrite: |
| 112 | logging.warning("Removing local copy %s", fpath) |
| 113 | os.remove(fpath) |
| 114 | else: |
| 115 | allGood = False |
| 116 | continue |
| 117 | _createDirIfNeeded(fpath) |
| 118 | _downloadFile(fpath, f.url) |
| 119 | if not _checkMD5(fpath, f.checksum): |
| 120 | logger.error("The downloaded file %s has a different checksum!", fpath) |
| 121 | allGood = False |
| 122 | |
| 123 | return allGood |
| 124 | |
| 125 | |
| 126 | def _parseArgs(): |
no test coverage detected