(url, download_dir, target_dir_name, sha1_hash = None, force_download = False, user_agent = None)
| 322 | return hasher.hexdigest() |
| 323 | |
| 324 | def downloadFile(url, download_dir, target_dir_name, sha1_hash = None, force_download = False, user_agent = None): |
| 325 | if not os.path.isdir(download_dir): |
| 326 | os.mkdir(download_dir) |
| 327 | |
| 328 | p = urlparse(url) |
| 329 | url = urlunparse([p[0], p[1], quote(p[2]), p[3], p[4], p[5]]) # replace special characters in the URL path |
| 330 | |
| 331 | filename_rel = os.path.split(p.path)[1] # get original filename |
| 332 | target_filename = os.path.join(download_dir, filename_rel) |
| 333 | |
| 334 | # check SHA1 hash, if file already exists |
| 335 | if os.path.exists(target_filename) and sha1_hash is not None and sha1_hash != "": |
| 336 | hash_file = computeFileHash(target_filename) |
| 337 | if hash_file != sha1_hash: |
| 338 | log("Hash of " + target_filename + " (" + hash_file + ") does not match expected hash (" + sha1_hash + "); forcing download") |
| 339 | force_download = True |
| 340 | |
| 341 | # download file |
| 342 | if (not os.path.exists(target_filename)) or force_download: |
| 343 | log("Downloading " + url + " to " + target_filename) |
| 344 | if p.scheme == "ssh": |
| 345 | downloadSCP(p.hostname, p.username, p.path, download_dir) |
| 346 | else: |
| 347 | if user_agent is not None: |
| 348 | opener = urllib.request.build_opener() |
| 349 | opener.addheaders = [('User-agent', user_agent)] |
| 350 | f = open(target_filename, 'wb') |
| 351 | f.write(opener.open(url).read()) |
| 352 | f.close() |
| 353 | else: |
| 354 | urlretrieve(url, target_filename) |
| 355 | else: |
| 356 | log("Skipping download of " + url + "; already downloaded") |
| 357 | |
| 358 | # check SHA1 hash |
| 359 | if sha1_hash is not None and sha1_hash != "": |
| 360 | hash_file = computeFileHash(target_filename) |
| 361 | if hash_file != sha1_hash: |
| 362 | raise RuntimeError("Hash of " + target_filename + " (" + hash_file + ") differs from expected hash (" + sha1_hash + ")") |
| 363 | |
| 364 | return target_filename |
| 365 | |
| 366 | |
| 367 | def downloadAndExtractFile(url, download_dir, target_dir_name, sha1_hash = None, force_download = False, user_agent = None): |
no test coverage detected