Method tokenize

llama_cpp/llama_tokenizer.py:49–52

(
        self, text: bytes, add_bos: bool = True, special: bool = True
    )

        self._model = llama._model  # type: ignore

    def tokenize(
        self, text: bytes, add_bos: bool = True, special: bool = True
    ) -> List[int]:
        return self._model.tokenize(text, add_bos=add_bos, special=special)

    def detokenize(
        self,