Download a file with curl, skipping if it already exists.
(url, dest_path)
| 73 | |
| 74 | |
| 75 | def download_file(url, dest_path): |
| 76 | """Download a file with curl, skipping if it already exists.""" |
| 77 | if os.path.exists(dest_path): |
| 78 | logger.info("Already exists, skipping: %s", dest_path) |
| 79 | return |
| 80 | logger.info("Downloading %s -> %s", url, dest_path) |
| 81 | tmp_path = dest_path + ".tmp" |
| 82 | result = subprocess.run( |
| 83 | ["curl", "-fL", "-o", tmp_path, url], |
| 84 | ) |
| 85 | if result.returncode != 0: |
| 86 | if os.path.exists(tmp_path): |
| 87 | os.unlink(tmp_path) |
| 88 | raise RuntimeError(f"Failed to download {url}") |
| 89 | os.rename(tmp_path, dest_path) |
| 90 | size_mb = os.path.getsize(dest_path) / (1024 * 1024) |
| 91 | logger.info(" -> %.1f MB", size_mb) |
| 92 | |
| 93 | |
| 94 | def decompress_zst(zst_path): |