Method DecodeTokens

data_utils/tokenization.py:1265–1270

(self, Tokens, type_token=False)

        return text

    def DecodeTokens(self, Tokens, type_token=False):
        if type_token:
            return ' '.join(t.token if isinstance(t, TypeToken) else t for t in Tokens)
        if isinstance(Tokens, Tokenization):
            Tokens = Tokens.tokenization
        return self.text_tokenizer.decode([self.TokenToId(tok) for tok in Tokens])