(self, text)
| 87 | return text |
| 88 | |
| 89 | def tokenize(self, text): |
| 90 | bpe_tokens = [] |
| 91 | bpe_tokens.extend(bpe_token for bpe_token in self.bpe(text).split(' ')) |
| 92 | return bpe_tokens |
| 93 | |
| 94 | def convert_tokens_to_ids(self, tokens): |
| 95 | return [self.encoder.get(token, 1) for token in tokens] |