| 36 | |
| 37 | |
| 38 | def upload_directory_to_s3(local_directory, destination, bucket): |
| 39 | import boto3 |
| 40 | |
| 41 | client = boto3.client("s3") |
| 42 | futures = [] |
| 43 | |
| 44 | with ThreadPoolExecutor(max_workers=5) as executor: |
| 45 | for root, dirs, files in os.walk(local_directory): |
| 46 | for filename in files: |
| 47 | local_path = os.path.join(root, filename) |
| 48 | relative_path = os.path.relpath(local_path, local_directory) |
| 49 | s3_path = os.path.join(destination, relative_path) |
| 50 | futures.append( |
| 51 | executor.submit(upload_file, client, local_path, bucket, s3_path) |
| 52 | ) |
| 53 | |
| 54 | for future in as_completed(futures): |
| 55 | future.result() |
| 56 | |
| 57 | |
| 58 | def download_directory_from_s3(bucket, s3_prefix, local_directory): |