Method tokenize

llama_cpp/llama_tokenizer.py:81–86

(
        self, text: bytes, add_bos: bool = True, special: bool = True
    )

        self.hf_tokenizer = hf_tokenizer

    def tokenize(
        self, text: bytes, add_bos: bool = True, special: bool = True
    ) -> List[int]:
        return self.hf_tokenizer.encode(
            text.decode("utf-8", errors="ignore"), add_special_tokens=special
        )

    def detokenize(
        self,