样本加载类
| 72 | |
| 73 | |
| 74 | class InputData(object): |
| 75 | """ |
| 76 | 样本加载类 |
| 77 | """ |
| 78 | |
| 79 | def __init__(self): |
| 80 | self.data = [] |
| 81 | self.test_data = [] |
| 82 | self.pos = 0 |
| 83 | self.word_vector_dict, self.word_id_dict = load_vectors(VECTORS_BIN) |
| 84 | self.dim_info = DimInfo() |
| 85 | self.maps = Maps() |
| 86 | _, self.dim_info.vec_dim = get_words_sizes(VECTORS_BIN) |
| 87 | self.dim_info.x_dim = len(self.word_vector_dict) * self.dim_info.vec_dim |
| 88 | self.maps.local_word_id_map = {} |
| 89 | |
| 90 | def clear_word_vector(self): |
| 91 | """ |
| 92 | 清理点内存 |
| 93 | """ |
| 94 | self.word_vector_dict.clear() |
| 95 | self.word_id_dict.clear() |
| 96 | |
| 97 | @staticmethod |
| 98 | def read_data_sets(file_name): |
| 99 | """ |
| 100 | 读取文件，加载数据 |
| 101 | """ |
| 102 | instance = InputData() |
| 103 | file_object = open(file_name, 'r') |
| 104 | while True: |
| 105 | line = file_object.readline(1024) |
| 106 | if line: |
| 107 | line = line.strip() |
| 108 | if len(line) == 0: |
| 109 | continue |
| 110 | split = line.split(' ') |
| 111 | group_id = 0 |
| 112 | try: |
| 113 | group_id = int(split[0]) |
| 114 | except ValueError: |
| 115 | continue |
| 116 | txt = ' '.join(split[1:]) |
| 117 | txt = txt.replace('None', '').strip() |
| 118 | if len(txt) == 0: |
| 119 | continue |
| 120 | |
| 121 | vectors = {} |
| 122 | seg_list = jieba.cut(txt) |
| 123 | for seg in seg_list: |
| 124 | seg_unicode = seg.encode('utf-8') |
| 125 | if seg_unicode in instance.word_vector_dict: |
| 126 | word_id = instance.word_id_dict[seg_unicode] |
| 127 | if word_id in instance.maps.local_word_id_map: |
| 128 | local_word_id = instance.maps.local_word_id_map[word_id] |
| 129 | vectors[local_word_id] = instance.word_vector_dict[seg_unicode] |
| 130 | else: |
| 131 | local_word_id = instance.dim_info.max_word_id |