| 43 | |
| 44 | |
| 45 | class LlamaTokenizer(BaseLlamaTokenizer): |
| 46 | def __init__(self, llama: llama_cpp.Llama): |
| 47 | self._model = llama._model # type: ignore |
| 48 | |
| 49 | def tokenize( |
| 50 | self, text: bytes, add_bos: bool = True, special: bool = True |
| 51 | ) -> List[int]: |
| 52 | return self._model.tokenize(text, add_bos=add_bos, special=special) |
| 53 | |
| 54 | def detokenize( |
| 55 | self, |
| 56 | tokens: List[int], |
| 57 | prev_tokens: Optional[List[int]] = None, |
| 58 | special: bool = False, |
| 59 | ) -> bytes: |
| 60 | return self._model.detokenize(tokens, special=special) |
| 61 | |
| 62 | def encode( |
| 63 | self, text: str, add_bos: bool = True, special: bool = True |
| 64 | ) -> List[int]: |
| 65 | return self.tokenize( |
| 66 | text.encode("utf-8", errors="ignore"), add_bos=add_bos, special=special |
| 67 | ) |
| 68 | |
| 69 | def decode(self, tokens: List[int]) -> str: |
| 70 | return self.detokenize(tokens).decode("utf-8", errors="ignore") |
| 71 | |
| 72 | @classmethod |
| 73 | def from_ggml_file(cls, path: str) -> "LlamaTokenizer": |
| 74 | return cls(llama_cpp.Llama(model_path=path, vocab_only=True)) |
| 75 | |
| 76 | |
| 77 | class LlamaHFTokenizer(BaseLlamaTokenizer): |