bpe encoding
(self, text)
| 209 | return word |
| 210 | |
| 211 | def encode(self, text): |
| 212 | """bpe encoding""" |
| 213 | bpe_tokens = [] |
| 214 | for token in re.findall(self.pat, text): |
| 215 | token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8')) |
| 216 | bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' ')) |
| 217 | return bpe_tokens |
| 218 | |
| 219 | def decode(self, tokens): |
| 220 | """bpe decoding""" |