download from S3 to local folder Args: fromPath (str): S3 URL toPath (str): local folder
(self,fromPath,toPath)
| 209 | self.copy_s3_file(f,bucket,fnew) |
| 210 | |
| 211 | def download_s3(self,fromPath,toPath): |
| 212 | """download from S3 to local folder |
| 213 | |
| 214 | Args: |
| 215 | fromPath (str): S3 URL |
| 216 | toPath (str): local folder |
| 217 | """ |
| 218 | if fromPath.startswith("s3n://"): |
| 219 | noSchemePath = fromPath[6:] |
| 220 | elif fromPath.startswith("s3://"): |
| 221 | noSchemePath = fromPath[5:] |
| 222 | parts = noSchemePath.split('/') |
| 223 | bucket = parts[0] |
| 224 | s3path = noSchemePath[len(bucket)+1:] |
| 225 | if self.aws_key: |
| 226 | self.conn = boto.connect_s3(self.aws_key,self.aws_secret) |
| 227 | else: |
| 228 | self.conn = boto.connect_s3() |
| 229 | b = self.conn.get_bucket(bucket) |
| 230 | for k in b.list(prefix=s3path): |
| 231 | if not k.name.endswith("/"): |
| 232 | basename = os.path.basename(k.name) |
| 233 | fnew = toPath+"/"+basename |
| 234 | logger.info("copying %s to %s",k.name,fnew) |
| 235 | k.get_contents_to_filename(fnew) |
| 236 | |
| 237 | |
| 238 | def copy(self,fromPath,toPath): |