(self, text)
| 209 | self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) |
| 210 | |
| 211 | def tokenize(self, text): |
| 212 | split_tokens = [] |
| 213 | for token in self.basic_tokenizer.tokenize(text): |
| 214 | for sub_token in self.wordpiece_tokenizer.tokenize(token): |
| 215 | split_tokens.append(sub_token) |
| 216 | return split_tokens |
| 217 | |
| 218 | def convert_tokens_to_ids(self, tokens): |
| 219 | """Converts a sequence of tokens into ids using the vocab.""" |