Method decode

lmdeploy/tokenizer.py:485–503

De-tokenize. Args: t: a list of token ids. offset: for incrementally decoding. Default to None, which means not applied. skip_special_tokens: Whether or not to remove special tokens in the decoding. Returns:

(
        self,
        t: Sequence[int],
        offset: int | None = None,
        skip_special_tokens: bool = True,
    )

        return encoded

    def decode(
        self,
        t: Sequence[int],
        offset: int | None = None,
        skip_special_tokens: bool = True,
    ):
        """De-tokenize.

        Args:
            t: a list of token ids.
            offset: for incrementally decoding. Default to None, which
                means not applied.
            skip_special_tokens: Whether or not to remove special
                tokens in the decoding.

        Returns:
            str: text of decoding tokens.
        """
        return self.model.decode(t, offset, skip_special_tokens)

    def detokenize_incrementally(self,
                                 all_input_ids: Sequence[int],