Save dataframe Parameters ---------- df : pandas daraframe dataframe to save location : str external filesystem location to save to df_format : str format to use : json or csv csv_index : bool whether to sa
(self,df,location,df_format="json",csv_index=True)
| 52 | # |
| 53 | |
| 54 | def save_dataframe(self,df,location,df_format="json",csv_index=True): |
| 55 | """Save dataframe |
| 56 | |
| 57 | Parameters |
| 58 | ---------- |
| 59 | |
| 60 | df : pandas daraframe |
| 61 | dataframe to save |
| 62 | location : str |
| 63 | external filesystem location to save to |
| 64 | df_format : str |
| 65 | format to use : json or csv |
| 66 | csv_index : bool |
| 67 | whether to save index when outputing to csv |
| 68 | """ |
| 69 | self.create_work_folder() |
| 70 | tmp_file = self.work_folder+"/df_tmp" |
| 71 | if df_format == 'csv': |
| 72 | logger.info("saving dataframe as csv") |
| 73 | df.to_csv(tmp_file,index=csv_index) |
| 74 | else: |
| 75 | logger.info("saving dataframe as json") |
| 76 | f = open(tmp_file,"w") |
| 77 | for i in range(0, df.shape[0]): |
| 78 | row = df.irow(i).dropna() |
| 79 | jNew = row.to_dict() |
| 80 | jStr = json.dumps(jNew,sort_keys=True) |
| 81 | f.write(jStr+"\n") |
| 82 | f.close() |
| 83 | futil = fu.FileUtil(aws_key=self.aws_key,aws_secret=self.aws_secret) |
| 84 | futil.copy(tmp_file,location) |
| 85 | |
| 86 | # |
| 87 | # download data and convert to dataframe |