copy from local file to S3 Args: fromPath (str): local file bucket (str): S3 bucket path (str): S3 prefix to add to files
(self,fromPath,bucket,path)
| 154 | |
| 155 | |
| 156 | def copy_s3_file(self,fromPath,bucket,path): |
| 157 | """copy from local file to S3 |
| 158 | |
| 159 | Args: |
| 160 | fromPath (str): local file |
| 161 | bucket (str): S3 bucket |
| 162 | path (str): S3 prefix to add to files |
| 163 | """ |
| 164 | if self.aws_key: |
| 165 | self.conn = boto.connect_s3(self.aws_key,self.aws_secret) |
| 166 | else: |
| 167 | self.conn = boto.connect_s3() |
| 168 | b = self.conn.get_bucket(bucket) |
| 169 | source_size = os.stat(fromPath).st_size |
| 170 | # Create a multipart upload request |
| 171 | uploadPath = path |
| 172 | logger.info("uploading to bucket %s path %s",bucket,uploadPath) |
| 173 | mp = b.initiate_multipart_upload(uploadPath) |
| 174 | chunk_size = 10485760 |
| 175 | chunk_count = int(math.ceil(source_size / float(chunk_size))) |
| 176 | for i in range(chunk_count): |
| 177 | offset = chunk_size * i |
| 178 | bytes = min(chunk_size, source_size - offset) |
| 179 | with FileChunkIO(fromPath, 'r', offset=offset,bytes=bytes) as fp: |
| 180 | logger.info("uploading to s3 chunk %d/%d",(i+1),chunk_count) |
| 181 | mp.upload_part_from_file(fp, part_num=i + 1) |
| 182 | # Finish the upload |
| 183 | logger.info("completing transfer to s3") |
| 184 | mp.complete_upload() |
| 185 | |
| 186 | |
| 187 |