Method decode

gpu/tokenizer.py:202–207

(self, tokens: List[int])

        self.eot_id = tokenizer.special_tokens["<|eot_id|>"]
    
    def decode(self, tokens: List[int]) -> str:
        # Decode the tokens to a string.
        decoded_str = self.tokenizer.decode(tokens)
        # Remove the special tokens from the decoded string.
        decoded_str = decoded_str.replace("<|eot_id|>", "")
        return decoded_str

    def encode_header(self, message: Message) -> List[int]:
        tokens = []