Method tokenize

llama_cpp/llama.py:595–611

Tokenize a string. Args: text: The utf-8 encoded string to tokenize. add_bos: Whether to add a beginning of sequence token. special: Whether to tokenize special tokens. Raises: RuntimeError: If the tokenization failed. Return

(
        self, text: bytes, add_bos: bool = True, special: bool = False
    )

        )

    def tokenize(
        self, text: bytes, add_bos: bool = True, special: bool = False
    ) -> List[int]:
        """Tokenize a string.

        Args:
            text: The utf-8 encoded string to tokenize.
            add_bos: Whether to add a beginning of sequence token.
            special: Whether to tokenize special tokens.

        Raises:
            RuntimeError: If the tokenization failed.

        Returns:
            A list of tokens.
        """
        return self.tokenizer_.tokenize(text, add_bos, special)

    def detokenize(
        self,