Method decode

NLP/UNIMO/src/model/tokenization.py:222–232

bpe decoding

(self, tokens)

        return bpe_tokens

    def decode(self, tokens):
        """bpe decoding"""
        decoded_tokens = []
        for token in tokens:
            if self.int_token.match(str(token)):
                decoded_tokens.append(self.decoder[int(token)])
            else:
                decoded_tokens.append(str(token))
        text = ''.join(decoded_tokens)
        text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors=self.errors)
        return text

    def decode_token(self, token):
        """decode one token"""