(self, tokens)
| 139 | return bpe_tokens |
| 140 | |
| 141 | def decode(self, tokens): |
| 142 | text = ''.join([self.decoder[token] for token in tokens]) |
| 143 | text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors=self.errors) |
| 144 | return text |
| 145 | |
| 146 | def get_encoder(vocab_file, bpe_file): |
| 147 | with open(vocab_file, 'r') as f: |
no outgoing calls