| 86 | ) |
| 87 | |
| 88 | def detokenize( |
| 89 | self, |
| 90 | tokens: List[int], |
| 91 | prev_tokens: Optional[List[int]] = None, |
| 92 | special: bool = False, |
| 93 | ) -> bytes: |
| 94 | skip_special_tokens = not special |
| 95 | if prev_tokens is not None: |
| 96 | text = self.hf_tokenizer.decode( |
| 97 | prev_tokens + tokens, skip_special_tokens=skip_special_tokens |
| 98 | ).encode("utf-8", errors="ignore") |
| 99 | prev_text = self.hf_tokenizer.decode( |
| 100 | prev_tokens, skip_special_tokens=skip_special_tokens |
| 101 | ).encode("utf-8", errors="ignore") |
| 102 | return text[len(prev_text) :] |
| 103 | else: |
| 104 | return self.hf_tokenizer.decode( |
| 105 | tokens, skip_special_tokens=skip_special_tokens |
| 106 | ).encode("utf-8", errors="ignore") |
| 107 | |
| 108 | @classmethod |
| 109 | def from_pretrained(cls, pretrained_model_name_or_path: str) -> "LlamaHFTokenizer": |