Method decode

lit_gpt/tokenizer.py:148–163

(
        self,
        tensor: Union[torch.Tensor, list],
        skip_special_tokens: Optional[bool] = None,
    )

        return torch.tensor(tokens, dtype=torch.int, device=device)

    def decode(
        self,
        tensor: Union[torch.Tensor, list],
        skip_special_tokens: Optional[bool] = None,
    ) -> str:
        if self.backend != HUGGINGFACE and skip_special_tokens is not None:
            print(f'WARN: Using {self.backend} does not allow `skip_special_tokens`.')

        tokens = tensor
        if isinstance(tensor, torch.Tensor):
            tokens = [tensor.item()] if tensor.ndim == 0 else tensor.tolist()

        if skip_special_tokens is not None and self.backend == HUGGINGFACE:
            return self.processor.decode(tokens, skip_special_tokens)

        return self.processor.decode(tokens)