Performs any necessary transformations before tokenization. This method should pop the arguments from kwargs and return kwargs as well. We test kwargs at the end of the encoding process to be sure all the arguments have been used.
(self, text: str, is_pretokenized=False, **kwargs)
| 621 | return batch_outputs |
| 622 | |
| 623 | def prepare_for_tokenization(self, text: str, is_pretokenized=False, **kwargs) -> (str, dict): |
| 624 | """ Performs any necessary transformations before tokenization. |
| 625 | |
| 626 | This method should pop the arguments from kwargs and return kwargs as well. |
| 627 | We test kwargs at the end of the encoding process to be sure all the arguments have been used. |
| 628 | """ |
| 629 | return (text, kwargs) |
| 630 | |
| 631 | def get_special_tokens_mask( |
| 632 | self, token_ids_0: List, token_ids_1: Optional[List] = None, already_has_special_tokens: bool = False |