Method encode

NLP/UNIMO/src/model/tokenization.py:214–220

bpe encoding

(self, text)

        return word

    def encode(self, text):
        """bpe encoding"""
        bpe_tokens = []
        for token in re.findall(self.pat, text):
            token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))
            bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' '))
        return bpe_tokens

    def decode(self, tokens):
        """bpe decoding"""