(self, tokens)
| 115 | return bpe_tokens, tokens |
| 116 | |
| 117 | def decode(self, tokens): |
| 118 | text = ''.join([self.decoder[token] for token in tokens]) |
| 119 | text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors=self.errors) |
| 120 | return text |
| 121 | |
| 122 | |
| 123 | def get_encoder(models_dir): |
no outgoing calls
no test coverage detected