Gather all CSV files in the input directory, concatenate them, and write to the output file. If the input directory contains multiple rows in a CSV file, only the last row will be saved. If the output file already exists, it will be overwritten.
(input_root: str, output_file: str)
| 25 | os.remove(osp.join(input_root, file)) |
| 26 | |
| 27 | def gather_csv_and_write(input_root: str, output_file: str): |
| 28 | """ |
| 29 | Gather all CSV files in the input directory, concatenate them, and write to the output file. |
| 30 | If the input directory contains multiple rows in a CSV file, only the last row will be saved. |
| 31 | If the output file already exists, it will be overwritten. |
| 32 | """ |
| 33 | seq_dfs = [] |
| 34 | for seq_csv_file in sorted(os.listdir(input_root)): |
| 35 | if seq_csv_file.endswith(".csv"): |
| 36 | df = pd.read_csv(osp.join(input_root, seq_csv_file)) |
| 37 | if len(df) > 1: |
| 38 | print(f"Warning: {osp.join(input_root, seq_csv_file)} has more than one row, only the last row will be saved.") |
| 39 | df = df.tail(1) |
| 40 | seq_dfs.append(df) |
| 41 | |
| 42 | if len(seq_dfs) == 0: |
| 43 | raise ValueError(f"No CSV files found in {input_root}. Returning an empty DataFrame.") |
| 44 | |
| 45 | df = pd.concat(seq_dfs, ignore_index=True) |
| 46 | if osp.isfile(output_file): |
| 47 | print(f"Warning: {output_file} already exists, data will be overwritten.") |
| 48 | df.to_csv(output_file, index=False) |
| 49 | return df |
| 50 | |
| 51 | def write_csv(file_path: str, data_dict: dict): |
| 52 | # transform data of one row to DataFrame |