Initialize vocabulary with tokens from token_generator.
(self, token_generator, add_reserved_tokens=True)
| 259 | self._init_vocab(token_gen()) |
| 260 | |
| 261 | def _init_vocab(self, token_generator, add_reserved_tokens=True): |
| 262 | """Initialize vocabulary with tokens from token_generator.""" |
| 263 | |
| 264 | self._id_to_token = {} |
| 265 | non_reserved_start_index = 0 |
| 266 | |
| 267 | if add_reserved_tokens: |
| 268 | self._id_to_token.update(enumerate(RESERVED_TOKENS)) |
| 269 | non_reserved_start_index = len(RESERVED_TOKENS) |
| 270 | |
| 271 | self._id_to_token.update( |
| 272 | enumerate(token_generator, start=non_reserved_start_index)) |
| 273 | |
| 274 | # _token_to_id is the reverse of _id_to_token |
| 275 | self._token_to_id = dict((v, k) |
| 276 | for k, v in six.iteritems(self._id_to_token)) |
| 277 | |
| 278 | def pad(self): |
| 279 | return self.pad_index |
no test coverage detected