获取训练样本
(self)
| 110 | print "finish" |
| 111 | |
| 112 | def next_batch(self): |
| 113 | """获取训练样本""" |
| 114 | XY = [] # lstm的训练输入 |
| 115 | Y = [] # lstm的训练输出 |
| 116 | EOS = [np.ones(self.word_vec_dim)] |
| 117 | sample_file_object = open('./samples/1', 'r') |
| 118 | lines = sample_file_object.readlines() |
| 119 | for line in lines: |
| 120 | line = line.strip() |
| 121 | split = line.split('|') |
| 122 | if len(split) == 2: |
| 123 | question = split[0] |
| 124 | answer = split[1] |
| 125 | print('question:[%s] answer:[%s]' % (question, answer)) |
| 126 | |
| 127 | good_sample = True |
| 128 | question_seq = [np.zeros(self.word_vec_dim)] * self.max_seq_len |
| 129 | answer_seq = [np.zeros(self.word_vec_dim)] * self.max_seq_len |
| 130 | answer_seq_one_hot = [np.zeros(self.one_hot_word_vectors_dim)] * self.max_seq_len |
| 131 | segments = jieba.cut(question) |
| 132 | for index, word in enumerate(segments): |
| 133 | if word in self.word_vector_dict: |
| 134 | vec = np.array(self.word_vector_dict[word]) / self.max_abs_weight |
| 135 | # 防止词过多越界 |
| 136 | if self.max_seq_len - index - 1 < 0: |
| 137 | good_sample = False |
| 138 | break |
| 139 | # 问题不足max_seq_len在前面补零，存储时倒序存储 |
| 140 | question_seq[self.max_seq_len - index - 1] = vec |
| 141 | else: |
| 142 | good_sample = False |
| 143 | |
| 144 | segments = jieba.cut(answer) |
| 145 | last_index = 0 |
| 146 | for index, word in enumerate(segments): |
| 147 | if word in self.word_vector_dict: |
| 148 | vec = np.array(self.word_vector_dict[word]) / self.max_abs_weight |
| 149 | # 防止词过多越界 |
| 150 | if index >= self.max_seq_len - 1: |
| 151 | good_sample = False |
| 152 | break |
| 153 | answer_seq[index] = vec |
| 154 | else: |
| 155 | good_sample = False |
| 156 | |
| 157 | if word in self.one_hot_word_vector_dict: |
| 158 | vec = self.one_hot_word_vector_dict[word] |
| 159 | answer_seq_one_hot[index] = vec |
| 160 | else: |
| 161 | good_sample = False |
| 162 | last_index = index |
| 163 | # 句子末尾加上EOS |
| 164 | answer_seq_one_hot[last_index + 1] = self.one_hot_word_vector_dict[self.eos_word] # EOS |
| 165 | |
| 166 | if good_sample: |
| 167 | xy = question_seq + EOS + answer_seq[0:-1] |
| 168 | y = answer_seq_one_hot |
| 169 | XY.append(xy) |