Get the DataFrames of each table in a folder Args: dataFolder: filepath to the folder with all tables Return: dataDfs (dict): key is the filename, value is the dataframe of that table
(dataFolder)
| 30 | |
| 31 | |
| 32 | def get_df(dataFolder): |
| 33 | ''' |
| 34 | Get the DataFrames of each table in a folder |
| 35 | Args: |
| 36 | dataFolder: filepath to the folder with all tables |
| 37 | Return: |
| 38 | dataDfs (dict): key is the filename, value is the dataframe of that table |
| 39 | ''' |
| 40 | |
| 41 | # dataFiles = glob.glob(dataFolder+"/*.csv") |
| 42 | dataFiles = os.listdir(dataFolder) |
| 43 | |
| 44 | dataDFs = {} |
| 45 | columns = 0 |
| 46 | |
| 47 | ind = 0 |
| 48 | for file in sorted(dataFiles): |
| 49 | ind += 1 |
| 50 | if file == "CSV0000000000000435.csv": continue |
| 51 | df = pd.read_csv(os.path.join(dataFolder, file), nrows=1000, encoding="ISO-8859-1", low_memory=False, lineterminator='\n') |
| 52 | # if len(df) > 1000: |
| 53 | # # get first 1000 rows |
| 54 | # df = df.head(1000) |
| 55 | filename = file.split("/")[-1] |
| 56 | dataDFs[filename] = df |
| 57 | |
| 58 | print(f"ind: {ind}, file: {file}, columns: {df.shape[1]}") |
| 59 | columns += df.shape[1] |
| 60 | |
| 61 | return dataDFs, columns |
| 62 | |
| 63 | |
| 64 | if __name__ == '__main__': |