Method encode_single_token

tiktoken/core.py:245–259

Encodes text corresponding to a single token to its token value. NOTE: this will encode all special tokens. Raises `KeyError` if the token is not in the vocabulary. ``` >>> enc.encode_single_token("hello") 31373 ```

(self, text_or_bytes: str | bytes)

        return self._core_bpe.encode_with_unstable(text, allowed_special)

    def encode_single_token(self, text_or_bytes: str | bytes) -> int:
        """Encodes text corresponding to a single token to its token value.

        NOTE: this will encode all special tokens.

        Raises `KeyError` if the token is not in the vocabulary.

        ```
        >>> enc.encode_single_token("hello")
        31373
        ```
        """
        if isinstance(text_or_bytes, str):
            text_or_bytes = text_or_bytes.encode("utf-8")
        return self._core_bpe.encode_single_token(text_or_bytes)

    # ====================
    # Decoding