| 203 | return dict(self.vocab, **self.added_tokens_encoder) |
| 204 | |
| 205 | def _tokenize(self, text): |
| 206 | split_tokens = [] |
| 207 | if self.do_basic_tokenize: |
| 208 | for token in self.basic_tokenizer.tokenize(text, never_split=self.all_special_tokens): |
| 209 | |
| 210 | # If the token is part of the never_split set |
| 211 | if token in self.basic_tokenizer.never_split: |
| 212 | split_tokens.append(token) |
| 213 | else: |
| 214 | split_tokens += self.wordpiece_tokenizer.tokenize(token) |
| 215 | else: |
| 216 | split_tokens = self.wordpiece_tokenizer.tokenize(text) |
| 217 | return split_tokens |
| 218 | |
| 219 | def _convert_token_to_id(self, token): |
| 220 | """ Converts a token (str) in an id using the vocab. """ |