upload from local path to S3 Args: fromPath (str): folder to copy from toPath (str): S3 URL
(self,fromPath,toPath)
| 186 | |
| 187 | |
| 188 | def upload_s3(self,fromPath,toPath): |
| 189 | """upload from local path to S3 |
| 190 | |
| 191 | Args: |
| 192 | fromPath (str): folder to copy from |
| 193 | toPath (str): S3 URL |
| 194 | """ |
| 195 | if toPath.startswith("s3n://"): |
| 196 | noSchemePath = toPath[6:] |
| 197 | elif toPath.startswith("s3://"): |
| 198 | noSchemePath = toPath[5:] |
| 199 | parts = noSchemePath.split('/') |
| 200 | bucket = parts[0] |
| 201 | opath = noSchemePath[len(bucket)+1:] |
| 202 | if os.path.isfile(fromPath): |
| 203 | self.copy_s3_file(fromPath,bucket,opath) |
| 204 | elif os.path.isdir(fromPath): |
| 205 | for f in glob.glob(fromPath+"/*"): |
| 206 | basename = os.path.basename(f) |
| 207 | fnew = opath+"/"+basename |
| 208 | logger.info("copying %s to %s",f,fnew) |
| 209 | self.copy_s3_file(f,bucket,fnew) |
| 210 | |
| 211 | def download_s3(self,fromPath,toPath): |
| 212 | """download from S3 to local folder |