(self, **kwargs)
| 154 | """ |
| 155 | |
| 156 | def __init__(self, **kwargs): |
| 157 | super().__init__(**kwargs) |
| 158 | |
| 159 | # Added tokens - We store this for both slow and fast tokenizers |
| 160 | # until the serialization of Fast tokenizers is updated |
| 161 | self.added_tokens_encoder: Dict[str, int] = {} |
| 162 | self.added_tokens_decoder: Dict[int, str] = {} |
| 163 | self.unique_no_split_tokens: List[str] = [] |
| 164 | |
| 165 | @property |
| 166 | def is_fast(self) -> bool: |
nothing calls this directly
no outgoing calls
no test coverage detected