| 44 | |
| 45 | |
| 46 | class WordVectorizer(object): |
| 47 | def __init__(self, meta_root, prefix): |
| 48 | vectors = np.load(pjoin(meta_root, '%s_data.npy'%prefix)) |
| 49 | words = pickle.load(open(pjoin(meta_root, '%s_words.pkl'%prefix), 'rb')) |
| 50 | word2idx = pickle.load(open(pjoin(meta_root, '%s_idx.pkl'%prefix), 'rb')) |
| 51 | self.word2vec = {w: vectors[word2idx[w]] for w in words} |
| 52 | |
| 53 | def _get_pos_ohot(self, pos): |
| 54 | pos_vec = np.zeros(len(POS_enumerator)) |
| 55 | if pos in POS_enumerator: |
| 56 | pos_vec[POS_enumerator[pos]] = 1 |
| 57 | else: |
| 58 | pos_vec[POS_enumerator['OTHER']] = 1 |
| 59 | return pos_vec |
| 60 | |
| 61 | def __len__(self): |
| 62 | return len(self.word2vec) |
| 63 | |
| 64 | def __getitem__(self, item): |
| 65 | word, pos = item.split('/') |
| 66 | if word in self.word2vec: |
| 67 | word_vec = self.word2vec[word] |
| 68 | vip_pos = None |
| 69 | for key, values in VIP_dict.items(): |
| 70 | if word in values: |
| 71 | vip_pos = key |
| 72 | break |
| 73 | if vip_pos is not None: |
| 74 | pos_vec = self._get_pos_ohot(vip_pos) |
| 75 | else: |
| 76 | pos_vec = self._get_pos_ohot(pos) |
| 77 | else: |
| 78 | word_vec = self.word2vec['unk'] |
| 79 | pos_vec = self._get_pos_ohot('OTHER') |
| 80 | return word_vec, pos_vec |
no outgoing calls
no test coverage detected