Method decode

codegeex/megatron/tokenizer/gpt2_tokenization.py:321–326

(self, tokens)

        return self.convert_tokens_to_ids(self.tokenize(text))

    def decode(self, tokens):
        text = "".join([self.decoder[token] for token in tokens])
        text = bytearray([self.byte_decoder[c] for c in text]).decode(
            "utf-8", errors=self.errors
        )
        return text

    def save_vocabulary(self, vocab_path):
        """Save the tokenizer vocabulary and merge files to a directory."""