Tokenize a prompt.
(self, s: str, add_bos: bool = True, add_special_tokens: bool = True, **kwargs)
| 365 | self.model._pad = __pad |
| 366 | |
| 367 | def encode(self, s: str, add_bos: bool = True, add_special_tokens: bool = True, **kwargs): |
| 368 | """Tokenize a prompt.""" |
| 369 | # ChtGLM4Tokenizer hardcode `add_speical_tokens=False` when tokenizing |
| 370 | # a prompt. Refer to https://huggingface.co/THUDM/glm-4-9b-chat/blob/main/tokenization_chatglm.py#L227 # noqa E501 |
| 371 | return super().encode(s, add_bos, add_special_tokens=False, **kwargs) |
| 372 | |
| 373 | |
| 374 | class ChatGLMTokenizer(HuggingFaceTokenizer): |