Useful for downloading a folder / zip file from dropbox/s3/cloudfront and unzipping it to path
(url, directory, warn_existing=True, overwrite=False)
| 149 | |
| 150 | |
| 151 | def download(url, directory, warn_existing=True, overwrite=False): |
| 152 | """Useful for downloading a folder / zip file from dropbox/s3/cloudfront and unzipping it to path""" |
| 153 | if has_stuff(directory, warn_existing, overwrite): |
| 154 | return |
| 155 | else: |
| 156 | os.makedirs(directory, exist_ok=True) |
| 157 | |
| 158 | log.info('Downloading %s to %s...', url, directory) |
| 159 | |
| 160 | request = requests.get(url, stream=True) |
| 161 | filename = url.split('/')[-1] |
| 162 | if '?' in filename: |
| 163 | filename = filename[:filename.index('?')] |
| 164 | location = os.path.join(tempfile.gettempdir(), filename) |
| 165 | with open(location, 'wb') as f: |
| 166 | if request.status_code == 404: |
| 167 | raise RuntimeError('Download URL not accessible %s' % url) |
| 168 | total_length = int(request.headers.get('content-length')) |
| 169 | for chunk in progress.bar(request.iter_content(chunk_size=1024), expected_size=(total_length / 1024) + 1): |
| 170 | if chunk: |
| 171 | f.write(chunk) |
| 172 | f.flush() |
| 173 | |
| 174 | log.info('done.') |
| 175 | zip_ref = zipfile.ZipFile(location, 'r') |
| 176 | log.info('Unzipping temp file %s to %s...', location, directory) |
| 177 | try: |
| 178 | zip_ref.extractall(directory) |
| 179 | print('done.') |
| 180 | except Exception: |
| 181 | print('You may want to close all programs that may have these files open or delete existing ' |
| 182 | 'folders this is trying to overwrite') |
| 183 | raise |
| 184 | finally: |
| 185 | zip_ref.close() |
| 186 | os.remove(location) |
| 187 | |
| 188 | |
| 189 | def dir_has_stuff(path): |