Tokenize a string.
(self, text)
| 269 | return word |
| 270 | |
| 271 | def tokenize(self, text): |
| 272 | """Tokenize a string.""" |
| 273 | bpe_tokens = [] |
| 274 | for token in re.findall(self.pat, text): |
| 275 | if sys.version_info[0] == 2: |
| 276 | token = "".join(self.byte_encoder[ord(b)] for b in token) |
| 277 | else: |
| 278 | token = "".join(self.byte_encoder[b] for b in token.encode("utf-8")) |
| 279 | bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(" ")) |
| 280 | return bpe_tokens |
| 281 | |
| 282 | def convert_tokens_to_ids(self, tokens): |
| 283 | """Converts a sequence of tokens into ids using the vocab.""" |