(self, text)
| 184 | self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) |
| 185 | |
| 186 | def tokenize(self, text): |
| 187 | split_tokens = [] |
| 188 | for token in self.basic_tokenizer.tokenize(text): |
| 189 | for sub_token in self.wordpiece_tokenizer.tokenize(token): |
| 190 | split_tokens.append(sub_token) |
| 191 | |
| 192 | return split_tokens |
| 193 | |
| 194 | def convert_tokens_to_ids(self, tokens): |
| 195 | return convert_by_vocab(self.vocab, tokens) |
no test coverage detected