Size of the full vocabulary with the added tokens
(self)
| 179 | return self.added_tokens_encoder |
| 180 | |
| 181 | def __len__(self): |
| 182 | """ Size of the full vocabulary with the added tokens """ |
| 183 | return self.vocab_size + len(self.added_tokens_encoder) |
| 184 | |
| 185 | def _add_tokens(self, new_tokens: Union[List[str], List[AddedToken]], special_tokens=False) -> int: |
| 186 | """ |
nothing calls this directly
no outgoing calls
no test coverage detected