| 581 | ] |
| 582 | |
| 583 | def __init__(self, verbose=True, **kwargs): |
| 584 | self._bos_token = None |
| 585 | self._eos_token = None |
| 586 | self._unk_token = None |
| 587 | self._sep_token = None |
| 588 | self._pad_token = None |
| 589 | self._cls_token = None |
| 590 | self._mask_token = None |
| 591 | self._pad_token_type_id = 0 |
| 592 | self._additional_special_tokens = [] |
| 593 | self.verbose = verbose |
| 594 | |
| 595 | # We directly set the hidden value to allow initialization with special tokens |
| 596 | # which are not yet in the vocabulary. Necesssary for serialization/de-serialization |
| 597 | # TODO clean this up at some point (probably by sitching to fast tokenizers) |
| 598 | for key, value in kwargs.items(): |
| 599 | if key in self.SPECIAL_TOKENS_ATTRIBUTES: |
| 600 | if key == "additional_special_tokens": |
| 601 | assert isinstance(value, (list, tuple)) and all(isinstance(t, str) for t in value) |
| 602 | setattr(self, key, value) |
| 603 | elif isinstance(value, (str, AddedToken)): |
| 604 | setattr(self, key, value) |
| 605 | else: |
| 606 | raise TypeError( |
| 607 | "special token {} has to be either str or AddedToken but got: {}".format(key, type(value)) |
| 608 | ) |
| 609 | |
| 610 | def sanitize_special_tokens(self) -> int: |
| 611 | """ Make sure that all the special tokens attributes of the tokenizer (tokenizer.mask_token, tokenizer.cls_token, ...) |