| 56 | |
| 57 | |
| 58 | def download_directory_from_s3(bucket, s3_prefix, local_directory): |
| 59 | import boto3 |
| 60 | |
| 61 | client = boto3.client("s3") |
| 62 | futures = [] |
| 63 | |
| 64 | with ThreadPoolExecutor(max_workers=5) as executor: |
| 65 | for key in client.list_objects(Bucket=bucket, Prefix=s3_prefix)["Contents"]: |
| 66 | # Get the relative path by removing the s3_prefix from the key |
| 67 | relative_path = key["Key"].replace(s3_prefix, "", 1).lstrip("/") |
| 68 | local_path = os.path.join(local_directory, relative_path) |
| 69 | |
| 70 | # Create the necessary directories |
| 71 | os.makedirs(os.path.dirname(local_path), exist_ok=True) |
| 72 | |
| 73 | futures.append( |
| 74 | executor.submit( |
| 75 | client.download_file, |
| 76 | bucket, |
| 77 | key["Key"], |
| 78 | local_path, |
| 79 | ) |
| 80 | ) |
| 81 | |
| 82 | for future in as_completed(futures): |
| 83 | future.result() |
| 84 | |
| 85 | |
| 86 | def require_auth(auth_credentials=Depends(HTTPBearer())): |