| 1 | # -*- coding: utf-8 -*- |
| 2 | import random |
| 3 | def read_write(source_file_path,target_file_path): |
| 4 | # 1.read file |
| 5 | source_file_object=open(source_file_path,mode='r') |
| 6 | target_file_object=open(target_file_path,mode='a') |
| 7 | |
| 8 | lines=source_file_object.readlines() |
| 9 | random.shuffle(lines) |
| 10 | # 2.write file |
| 11 | for i,line in enumerate(lines): |
| 12 | # line=w29368 w6 w7851 w24869 w35610 w111 __label__-5732564720446782766 2787171473654490487 |
| 13 | raw_list = line.strip().split("__label__") |
| 14 | input_list = raw_list[0].strip().split(" ") |
| 15 | input_list = [x.strip().replace(" ", "") for x in input_list if x != ''] |
| 16 | label_list = raw_list[1:] |
| 17 | label_list=label_list[0].split(" ") |
| 18 | label_list=[l.strip().replace(" ", "") for l in label_list if l != ''] |
| 19 | strings=" ".join(input_list)+' __label__'+" __label__".join(label_list)+"\n" |
| 20 | target_file_object.write(strings) |
| 21 | if i%10000==0: |
| 22 | print(i,strings) |
| 23 | target_file_object.close() |
| 24 | source_file_object.close() |
| 25 | #source_file_path='/Users/xuliang/Downloads/train-zhihu6-title-desc-folder.txt/train-zhihu6-title-desc.txt' |
| 26 | #target_file_path='/Users/xuliang/Downloads/train-zhihu6-title-desc-folder.txt/train-zhihu-title-desc-multiple-label-v6.txt' |
| 27 | |