(self, model_path, trust_remote_code: bool = False)
| 375 | """Tokenizer of GLM2.""" |
| 376 | |
| 377 | def __init__(self, model_path, trust_remote_code: bool = False): |
| 378 | super().__init__(model_path, trust_remote_code=trust_remote_code) |
| 379 | original_pad = self.model._pad |
| 380 | |
| 381 | def __pad(*args, **kwargs): |
| 382 | if 'padding_side' in kwargs: |
| 383 | kwargs.pop('padding_side') |
| 384 | return original_pad(*args, **kwargs) |
| 385 | |
| 386 | # fix for transformers>4.45.0 |
| 387 | self.model._pad = __pad |
| 388 | |
| 389 | |
| 390 | class GptOssTokenizer(HuggingFaceTokenizer): |