| 50 | |
| 51 | |
| 52 | def pre_multi_process(set_path, file_list, q, id): |
| 53 | # print("{} start".format(id)) |
| 54 | datas = {} |
| 55 | for i, sets_file in enumerate(file_list): |
| 56 | try: |
| 57 | df = pd.read_csv(sets_file, dtype='str', lineterminator='\n').dropna() |
| 58 | except pd.errors.ParserError as e: |
| 59 | print(sets_file) |
| 60 | columns = df.columns.tolist() |
| 61 | data = df.values.T.tolist() |
| 62 | for column, vals in zip(columns, data): |
| 63 | # 需要对value去重 |
| 64 | key = sets_file + "." + column |
| 65 | datas[key] = list(set(vals)) |
| 66 | if id==0: |
| 67 | sys.stdout.write("\rId 0 Process Read and pre {}/{} files".format(i+1, len(file_list))) |
| 68 | if id==0: |
| 69 | sys.stdout.write("\n") |
| 70 | q.put((datas,)) |
| 71 | |
| 72 | def split_list(lst, num_parts): |
| 73 | avg = len(lst) // num_parts |