Decode token IDs to text.
(self, tokens)
| 257 | return bpe_tokens |
| 258 | |
| 259 | def decode(self, tokens): |
| 260 | """Decode token IDs to text.""" |
| 261 | text = ''.join([self.decoder[token] for token in tokens]) |
| 262 | text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace").replace('</w>', ' ') |
| 263 | return text |
| 264 | |
| 265 | def __call__(self, texts: Union[str, List[str]], context_length: Optional[int] = None) -> torch.LongTensor: |
| 266 | """Returns the tokenized representation of given input string(s). |