Method encode

llama_cpp/llama_tokenizer.py:62–67

(
        self, text: str, add_bos: bool = True, special: bool = True
    )

        return self._model.detokenize(tokens, special=special)

    def encode(
        self, text: str, add_bos: bool = True, special: bool = True
    ) -> List[int]:
        return self.tokenize(
            text.encode("utf-8", errors="ignore"), add_bos=add_bos, special=special
        )

    def decode(self, tokens: List[int]) -> str:
        return self.detokenize(tokens).decode("utf-8", errors="ignore")