Method __init__

llama_cpp/llama_tokenizer.py:78–79

(self, hf_tokenizer: Any)

class LlamaHFTokenizer(BaseLlamaTokenizer):
    def __init__(self, hf_tokenizer: Any):
        self.hf_tokenizer = hf_tokenizer

    def tokenize(
        self, text: bytes, add_bos: bool = True, special: bool = True