(self, tokens)
| 270 | return self.convert_tokens_to_ids(self.tokenize(text)) |
| 271 | |
| 272 | def decode(self, tokens): |
| 273 | text = ''.join([self.decoder[token] for token in tokens]) |
| 274 | text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors=self.errors) |
| 275 | return text |
| 276 | |
| 277 | def save_vocabulary(self, vocab_path): |
| 278 | """Save the tokenizer vocabulary and merge files to a directory.""" |
nothing calls this directly
no outgoing calls
no test coverage detected