(self, text)
| 137 | return word |
| 138 | |
| 139 | def encode(self, text): |
| 140 | bpe_tokens = [] |
| 141 | text = whitespace_clean(basic_clean(text)).lower() |
| 142 | for token in re.findall(self.pat, text): |
| 143 | token = ''.join(self.byte_encoder[b] |
| 144 | for b in token.encode('utf-8')) |
| 145 | bpe_tokens.extend(self.encoder[bpe_token] |
| 146 | for bpe_token in self.bpe(token).split(' ')) |
| 147 | return bpe_tokens |
| 148 | |
| 149 | def decode(self, tokens): |
| 150 | text = ''.join([self.decoder[token] for token in tokens]) |