Method decode_tokens_bytes

tiktoken/core.py:303–310

Decodes a list of tokens into a list of bytes. Useful for visualising tokenisation. >>> enc.decode_tokens_bytes([31373, 995]) [b'hello', b' world']

(self, tokens: Sequence[int])

        return self._core_bpe.decode_single_token_bytes(token)

    def decode_tokens_bytes(self, tokens: Sequence[int]) -> list[bytes]:
        """Decodes a list of tokens into a list of bytes.

        Useful for visualising tokenisation.
        >>> enc.decode_tokens_bytes([31373, 995])
        [b'hello', b' world']
        """
        return [self.decode_single_token_bytes(token) for token in tokens]

    def decode_with_offsets(self, tokens: Sequence[int]) -> tuple[str, list[int]]:
        """Decodes a list of tokens into a string and a list of offsets.