| 49 | |
| 50 | |
| 51 | class WordVectorizer(object): |
| 52 | |
| 53 | def __init__(self, meta_root, prefix): |
| 54 | vectors = np.load(pjoin(meta_root, '%s_data.npy' % prefix)) |
| 55 | words = pickle.load( |
| 56 | open(pjoin(meta_root, '%s_words.pkl' % prefix), 'rb')) |
| 57 | word2idx = pickle.load( |
| 58 | open(pjoin(meta_root, '%s_idx.pkl' % prefix), 'rb')) |
| 59 | self.word2vec = {w: vectors[word2idx[w]] for w in words} |
| 60 | |
| 61 | def _get_pos_ohot(self, pos): |
| 62 | pos_vec = np.zeros(len(POS_enumerator)) |
| 63 | if pos in POS_enumerator: |
| 64 | pos_vec[POS_enumerator[pos]] = 1 |
| 65 | else: |
| 66 | pos_vec[POS_enumerator['OTHER']] = 1 |
| 67 | return pos_vec |
| 68 | |
| 69 | def __len__(self): |
| 70 | return len(self.word2vec) |
| 71 | |
| 72 | def __getitem__(self, item): |
| 73 | word, pos = item.split('/') |
| 74 | if word in self.word2vec: |
| 75 | word_vec = self.word2vec[word] |
| 76 | vip_pos = None |
| 77 | for key, values in VIP_dict.items(): |
| 78 | if word in values: |
| 79 | vip_pos = key |
| 80 | break |
| 81 | if vip_pos is not None: |
| 82 | pos_vec = self._get_pos_ohot(vip_pos) |
| 83 | else: |
| 84 | pos_vec = self._get_pos_ohot(pos) |
| 85 | else: |
| 86 | word_vec = self.word2vec['unk'] |
| 87 | pos_vec = self._get_pos_ohot('OTHER') |
| 88 | return word_vec, pos_vec |