| 25 | 2017-08-22 |
| 26 | """ |
| 27 | def TextProcessing(folder_path, test_size = 0.2): |
| 28 | folder_list = os.listdir(folder_path) #查看folder_path下的文件 |
| 29 | data_list = [] #数据集数据 |
| 30 | class_list = [] #数据集类别 |
| 31 | |
| 32 | #遍历每个子文件夹 |
| 33 | for folder in folder_list: |
| 34 | new_folder_path = os.path.join(folder_path, folder) #根据子文件夹，生成新的路径 |
| 35 | files = os.listdir(new_folder_path) #存放子文件夹下的txt文件的列表 |
| 36 | |
| 37 | j = 1 |
| 38 | #遍历每个txt文件 |
| 39 | for file in files: |
| 40 | if j > 100: #每类txt样本数最多100个 |
| 41 | break |
| 42 | with open(os.path.join(new_folder_path, file), 'r', encoding = 'utf-8') as f: #打开txt文件 |
| 43 | raw = f.read() |
| 44 | |
| 45 | word_cut = jieba.cut(raw, cut_all = False) #精简模式，返回一个可迭代的generator |
| 46 | word_list = list(word_cut) #generator转换为list |
| 47 | |
| 48 | data_list.append(word_list) #添加数据集数据 |
| 49 | class_list.append(folder) #添加数据集类别 |
| 50 | j += 1 |
| 51 | |
| 52 | data_class_list = list(zip(data_list, class_list)) #zip压缩合并，将数据与标签对应压缩 |
| 53 | random.shuffle(data_class_list) #将data_class_list乱序 |
| 54 | index = int(len(data_class_list) * test_size) + 1 #训练集和测试集切分的索引值 |
| 55 | train_list = data_class_list[index:] #训练集 |
| 56 | test_list = data_class_list[:index] #测试集 |
| 57 | train_data_list, train_class_list = zip(*train_list) #训练集解压缩 |
| 58 | test_data_list, test_class_list = zip(*test_list) #测试集解压缩 |
| 59 | |
| 60 | all_words_dict = {} #统计训练集词频 |
| 61 | for word_list in train_data_list: |
| 62 | for word in word_list: |
| 63 | if word in all_words_dict.keys(): |
| 64 | all_words_dict[word] += 1 |
| 65 | else: |
| 66 | all_words_dict[word] = 1 |
| 67 | |
| 68 | #根据键的值倒序排序 |
| 69 | all_words_tuple_list = sorted(all_words_dict.items(), key = lambda f:f[1], reverse = True) |
| 70 | all_words_list, all_words_nums = zip(*all_words_tuple_list) #解压缩 |
| 71 | all_words_list = list(all_words_list) #转换成列表 |
| 72 | return all_words_list, train_data_list, test_data_list, train_class_list, test_class_list |
| 73 | |
| 74 | """ |
| 75 | 函数说明:读取文件里的内容，并去重 |