Download file from specified URL link, support process bar and hash check. Args: url: source URL link to download file. filepath: target filepath to save the downloaded file (including the filename). If undefined, `os.path.basename(url)` will be used. ha
(
url: str,
filepath: PathLike = "",
hash_val: str | None = None,
hash_type: str = "md5",
progress: bool = True,
**gdown_kwargs: Any,
)
| 189 | |
| 190 | |
| 191 | def download_url( |
| 192 | url: str, |
| 193 | filepath: PathLike = "", |
| 194 | hash_val: str | None = None, |
| 195 | hash_type: str = "md5", |
| 196 | progress: bool = True, |
| 197 | **gdown_kwargs: Any, |
| 198 | ) -> None: |
| 199 | """ |
| 200 | Download file from specified URL link, support process bar and hash check. |
| 201 | |
| 202 | Args: |
| 203 | url: source URL link to download file. |
| 204 | filepath: target filepath to save the downloaded file (including the filename). |
| 205 | If undefined, `os.path.basename(url)` will be used. |
| 206 | hash_val: expected hash value to validate the downloaded file. |
| 207 | if None, skip hash validation. |
| 208 | hash_type: 'md5' or 'sha1', defaults to 'md5'. |
| 209 | progress: whether to display a progress bar. |
| 210 | gdown_kwargs: other args for `gdown` except for the `url`, `output` and `quiet`. |
| 211 | these args will only be used if download from google drive. |
| 212 | details of the args of it: |
| 213 | https://github.com/wkentaro/gdown/blob/main/gdown/download.py |
| 214 | |
| 215 | Raises: |
| 216 | RuntimeError: When the hash validation of the ``filepath`` existing file fails. |
| 217 | RuntimeError: When a network issue or denied permission prevents the |
| 218 | file download from ``url`` to ``filepath``. |
| 219 | URLError: See urllib.request.urlretrieve. |
| 220 | HTTPError: See urllib.request.urlretrieve. |
| 221 | ContentTooShortError: See urllib.request.urlretrieve. |
| 222 | IOError: See urllib.request.urlretrieve. |
| 223 | RuntimeError: When the hash validation of the ``url`` downloaded file fails. |
| 224 | |
| 225 | """ |
| 226 | if not filepath: |
| 227 | filepath = Path(".", _basename(url)).resolve() |
| 228 | logger.info(f"Default downloading to '{filepath}'") |
| 229 | filepath = Path(filepath) |
| 230 | if filepath.exists(): |
| 231 | if not check_hash(filepath, hash_val, hash_type): |
| 232 | raise RuntimeError( |
| 233 | f"{hash_type} check of existing file failed: filepath={filepath}, expected {hash_type}={hash_val}." |
| 234 | ) |
| 235 | logger.info(f"File exists: {filepath}, skipped downloading.") |
| 236 | return |
| 237 | try: |
| 238 | with tempfile.TemporaryDirectory() as tmp_dir: |
| 239 | tmp_name = Path(tmp_dir, _basename(filepath)) |
| 240 | if urlparse(url).netloc == "drive.google.com": |
| 241 | if not has_gdown: |
| 242 | raise RuntimeError("To download files from Google Drive, please install the gdown dependency.") |
| 243 | if "fuzzy" not in gdown_kwargs: |
| 244 | gdown_kwargs["fuzzy"] = True # default to true for flexible url |
| 245 | gdown.download(url, f"{tmp_name}", quiet=not progress, **gdown_kwargs) |
| 246 | elif urlparse(url).netloc == "cloud-api.yandex.net": |
| 247 | with urlopen(url) as response: |
| 248 | code = response.getcode() |
searching dependent graphs…