Tokenize a string.
(self, text)
| 249 | return output + bos_token_ids + token_ids_1 |
| 250 | |
| 251 | def _tokenize(self, text): |
| 252 | """Tokenize a string.""" |
| 253 | bpe_tokens = [] |
| 254 | for token in re.findall(self.pat, text): |
| 255 | token = "".join( |
| 256 | self.byte_encoder[b] for b in token.encode("utf-8") |
| 257 | ) # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case) |
| 258 | bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(" ")) |
| 259 | return bpe_tokens |
| 260 | |
| 261 | def _convert_token_to_id(self, token): |
| 262 | """Converts a token (str) in an id using the vocab.""" |