Encode text to token IDs.
(self, text)
| 248 | return word |
| 249 | |
| 250 | def encode(self, text): |
| 251 | """Encode text to token IDs.""" |
| 252 | bpe_tokens = [] |
| 253 | text = self.clean_fn(text) |
| 254 | for token in re.findall(self.pat, text): |
| 255 | token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8')) |
| 256 | bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' ')) |
| 257 | return bpe_tokens |
| 258 | |
| 259 | def decode(self, tokens): |
| 260 | """Decode token IDs to text.""" |