Method encode

data_utils/tokenization_gpt2.py:269–270

(self, text)

        return tokens

    def encode(self, text):
        return self.convert_tokens_to_ids(self.tokenize(text))

    def decode(self, tokens):
        text = ''.join([self.decoder[token] for token in tokens])