| 90 | |
| 91 | @registry.register('word_emb', 'bpemb') |
| 92 | class BPEmb(Embedder): |
| 93 | def __init__(self, dim, vocab_size, lang='en'): |
| 94 | self.bpemb = bpemb.BPEmb(lang=lang, dim=dim, vs=vocab_size) |
| 95 | self.dim = dim |
| 96 | self.vectors = torch.from_numpy(self.bpemb.vectors) |
| 97 | |
| 98 | def tokenize(self, text): |
| 99 | return self.bpemb.encode(text) |
| 100 | |
| 101 | def untokenize(self, tokens): |
| 102 | return self.bpemb.decode(tokens) |
| 103 | |
| 104 | def lookup(self, token): |
| 105 | i = self.bpemb.spm.PieceToId(token) |
| 106 | if i == self.bpemb.spm.unk_id(): |
| 107 | return None |
| 108 | return self.vectors[i] |
| 109 | |
| 110 | def contains(self, token): |
| 111 | return self.lookup(token) is not None |
| 112 | |
| 113 | def to(self, device): |
| 114 | self.vectors = self.vectors.to(device) |
nothing calls this directly
no outgoing calls
no test coverage detected