| 10 | |
| 11 | |
| 12 | class MyLSTM(object): |
| 13 | def __init__(self): |
| 14 | self.max_abs_weight = 32 # 最大权重绝对值，用来对词向量做正规化 |
| 15 | self.max_seq_len = 8 # 最大句子长度(词) |
| 16 | self.word_vec_dim = 0 # 词向量维度，读vectors.bin二进制时动态确定 |
| 17 | self.epoch = 1000 |
| 18 | self.word_vector_dict = {} # 词向量词典，加载vectors.bin读入 |
| 19 | self.one_hot_word_vector_dict = {} # 根据样本词汇生成的softmax用的词向量 |
| 20 | self.word_id_word_dict = {} |
| 21 | self.one_hot_word_vectors_dim = 1 # softmax用的词向量维度，从1开始，保留0作为EOS的word_id |
| 22 | self.eos_word_id = 0 |
| 23 | self.eos_word = 'EOS' |
| 24 | self.vectors_bin_file = './vectors.bin' # 词向量二进制 |
| 25 | self.model_dir = './model/model' # 模型文件路径 |
| 26 | self.n_hidden = 1000 # lstm隐藏状态单元数目 |
| 27 | self.learning_rate = 0.01 # 学习率 |
| 28 | |
| 29 | def load_one_hot_word_vectors(self): |
| 30 | |
| 31 | word_id_dict = {} |
| 32 | sample_file_object = open('./samples/1', 'r') |
| 33 | lines = sample_file_object.readlines() |
| 34 | for line in lines: |
| 35 | line = line.strip() |
| 36 | split = line.split('|') |
| 37 | if len(split) == 2: |
| 38 | answer = split[1] |
| 39 | segments = jieba.cut(answer) |
| 40 | for word in segments: |
| 41 | if word not in word_id_dict: |
| 42 | word_id_dict[word] = self.one_hot_word_vectors_dim |
| 43 | self.word_id_word_dict[self.one_hot_word_vectors_dim] = word |
| 44 | self.one_hot_word_vectors_dim = self.one_hot_word_vectors_dim + 1 |
| 45 | |
| 46 | # 添加一个结尾符 |
| 47 | vector = np.zeros(self.one_hot_word_vectors_dim) |
| 48 | vector[self.eos_word_id] = 1 |
| 49 | self.one_hot_word_vector_dict[self.eos_word] = vector |
| 50 | self.word_id_word_dict[self.eos_word_id] = self.eos_word |
| 51 | |
| 52 | for line in lines: |
| 53 | line = line.strip() |
| 54 | split = line.split('|') |
| 55 | if len(split) == 2: |
| 56 | answer = split[1] |
| 57 | segments = jieba.cut(answer) |
| 58 | for word in segments: |
| 59 | if word not in self.one_hot_word_vector_dict: |
| 60 | word_id = word_id_dict[word] |
| 61 | print word, word_id |
| 62 | vector = np.zeros(self.one_hot_word_vectors_dim) |
| 63 | vector[word_id] = 1 |
| 64 | self.one_hot_word_vector_dict[word] = vector |
| 65 | |
| 66 | sample_file_object.close() |
| 67 | |
| 68 | def load_word_vectors(self): |
| 69 | """加载词向量二进制到内存""" |