(url, md5, dataset_name, train_file, test_file, cache=False)
| 84 | |
| 85 | |
| 86 | def _download_dataset(url, md5, dataset_name, train_file, test_file, cache=False): |
| 87 | # TODO(yazevnul): this is not thread safe (or process safe?), we should take a file lock when |
| 88 | # enter this function to avoid dataset being overwritten or corrupted or something else that may |
| 89 | # have happen when OS operated simultaneously on the same file. Same thing should probably be |
| 90 | # done with `_cached_download`. |
| 91 | dir_path = os.path.join(_get_cache_path(), dataset_name) if cache else tempfile.mkdtemp() |
| 92 | train_path = os.path.join(dir_path, train_file) |
| 93 | test_path = os.path.join(dir_path, test_file) |
| 94 | if not (os.path.exists(train_path) and os.path.exists(test_path)): |
| 95 | _ensure_dir_exists(dir_path) |
| 96 | file_descriptor, file_path = tempfile.mkstemp() |
| 97 | os.close(file_descriptor) |
| 98 | try: |
| 99 | _cached_download(url, md5, file_path) |
| 100 | _extract(file_path, dir_path) |
| 101 | finally: |
| 102 | os.remove(file_path) |
| 103 | # move files for safe delete of temp dir |
| 104 | if not cache: |
| 105 | fd_new_train, new_train_path = tempfile.mkstemp() |
| 106 | fd_new_test, new_test_path = tempfile.mkstemp() |
| 107 | os.close(fd_new_train) |
| 108 | os.close(fd_new_test) |
| 109 | os.replace(train_path, new_train_path) |
| 110 | os.replace(test_path, new_test_path) |
| 111 | shutil.rmtree(dir_path) |
| 112 | train_path, test_path = new_train_path, new_test_path |
| 113 | return train_path, test_path |
| 114 | |
| 115 | |
| 116 | def _load_dataset_pd(url, md5, dataset_name, train_file, test_file, sep=',', header='infer', cache=False): |
no test coverage detected