Method decode_single_token_bytes

tiktoken/core.py:289–301

Decodes a token into bytes. NOTE: this will decode all special tokens. Raises `KeyError` if the token is not in the vocabulary. ``` >>> enc.decode_single_token_bytes(31373) b'hello' ```

(self, token: int)

        return self._core_bpe.decode_bytes(tokens).decode("utf-8", errors=errors)

    def decode_single_token_bytes(self, token: int) -> bytes:
        """Decodes a token into bytes.

        NOTE: this will decode all special tokens.

        Raises `KeyError` if the token is not in the vocabulary.

        ```
        >>> enc.decode_single_token_bytes(31373)
        b'hello'
        ```
        """
        return self._core_bpe.decode_single_token_bytes(token)

    def decode_tokens_bytes(self, tokens: Sequence[int]) -> list[bytes]:
        """Decodes a list of tokens into a list of bytes.