Method tokenizer

llama_cpp/llama.py:2260–2262

Return the llama tokenizer for this model.

(self)

        return self._model.n_vocab()

    def tokenizer(self) -> LlamaTokenizer:
        """Return the llama tokenizer for this model."""
        return LlamaTokenizer(self)

    def token_eos(self) -> int:
        """Return the end-of-sequence token."""