加载词向量二进制到内存
(self)
| 66 | sample_file_object.close() |
| 67 | |
| 68 | def load_word_vectors(self): |
| 69 | """加载词向量二进制到内存""" |
| 70 | float_size = 4 # 一个浮点数4字节 |
| 71 | max_w = 50 # 最大单词字数 |
| 72 | input_file = open(self.vectors_bin_file, "rb") |
| 73 | # 获取词表数目及向量维度 |
| 74 | words_and_size = input_file.readline() |
| 75 | words_and_size = words_and_size.strip() |
| 76 | words = long(words_and_size.split(' ')[0]) |
| 77 | self.word_vec_dim = long(words_and_size.split(' ')[1]) |
| 78 | print("词表总词数：%d" % words) |
| 79 | print("词向量维度：%d" % self.word_vec_dim) |
| 80 | |
| 81 | for b in range(0, words): |
| 82 | a = 0 |
| 83 | word = '' |
| 84 | # 读取一个词 |
| 85 | while True: |
| 86 | c = input_file.read(1) |
| 87 | word = word + c |
| 88 | if False == c or c == ' ': |
| 89 | break |
| 90 | if a < max_w and c != '\n': |
| 91 | a = a + 1 |
| 92 | word = word.strip() |
| 93 | vector = [] |
| 94 | |
| 95 | for index in range(0, self.word_vec_dim): |
| 96 | m = input_file.read(float_size) |
| 97 | (weight,) = struct.unpack('f', m) |
| 98 | f_weight = float(weight) |
| 99 | vector.append(f_weight) |
| 100 | |
| 101 | # 将词及其对应的向量存到dict中 |
| 102 | try: |
| 103 | self.word_vector_dict[word.decode('utf-8')] = vector[0:self.word_vec_dim] |
| 104 | except: |
| 105 | # 异常的词舍弃掉 |
| 106 | # print('bad word:' + word) |
| 107 | pass |
| 108 | |
| 109 | input_file.close() |
| 110 | print "finish" |
| 111 | |
| 112 | def next_batch(self): |
| 113 | """获取训练样本""" |