Initialize tokens from a list of tokens. It is ok if reserved tokens appear in the vocab list. They will be removed. The set of tokens in vocab_list should be unique. Args: vocab_list: A list of tokens.
(self, vocab_list)
| 243 | self._init_vocab(token_gen(), add_reserved_tokens=False) |
| 244 | |
| 245 | def _init_vocab_from_list(self, vocab_list): |
| 246 | """Initialize tokens from a list of tokens. |
| 247 | |
| 248 | It is ok if reserved tokens appear in the vocab list. They will be |
| 249 | removed. The set of tokens in vocab_list should be unique. |
| 250 | |
| 251 | Args: |
| 252 | vocab_list: A list of tokens. |
| 253 | """ |
| 254 | def token_gen(): |
| 255 | for token in vocab_list: |
| 256 | if token not in RESERVED_TOKENS: |
| 257 | yield token |
| 258 | |
| 259 | self._init_vocab(token_gen()) |
| 260 | |
| 261 | def _init_vocab(self, token_generator, add_reserved_tokens=True): |
| 262 | """Initialize vocabulary with tokens from token_generator.""" |