Load vocab from a file. Args: filename: The file to load vocabulary from.
(self, filename)
| 228 | return self._id_to_token.get(idx, "ID_%d" % idx) |
| 229 | |
| 230 | def _init_vocab_from_file(self, filename): |
| 231 | """Load vocab from a file. |
| 232 | |
| 233 | Args: |
| 234 | filename: The file to load vocabulary from. |
| 235 | """ |
| 236 | with open(filename) as f: |
| 237 | tokens = [token.strip() for token in f.readlines()] |
| 238 | |
| 239 | def token_gen(): |
| 240 | for token in tokens: |
| 241 | yield token |
| 242 | |
| 243 | self._init_vocab(token_gen(), add_reserved_tokens=False) |
| 244 | |
| 245 | def _init_vocab_from_list(self, vocab_list): |
| 246 | """Initialize tokens from a list of tokens. |