Converts a sequence of tokens (string) in a single string.
(self, tokens)
| 267 | return self.decoder.get(index) |
| 268 | |
| 269 | def convert_tokens_to_string(self, tokens): |
| 270 | """Converts a sequence of tokens (string) in a single string.""" |
| 271 | text = "".join(tokens) |
| 272 | text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors) |
| 273 | return text |
| 274 | |
| 275 | def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]: |
| 276 | if not os.path.isdir(save_directory): |