tokenize the text
(self, text)
| 312 | self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) |
| 313 | |
| 314 | def tokenize(self, text): |
| 315 | """tokenize the text""" |
| 316 | split_tokens = [] |
| 317 | for token in self.basic_tokenizer.tokenize(text): |
| 318 | for sub_token in self.wordpiece_tokenizer.tokenize(token): |
| 319 | split_tokens.append(sub_token) |
| 320 | |
| 321 | return split_tokens |
| 322 | |
| 323 | def convert_tokens_to_ids(self, tokens): |
| 324 | """convert tokens to vocab ids""" |
no test coverage detected