Method tok_encode

test/general/lm_eval/models/gpt3.py:111–112

(self, string: str)

        raise NotImplementedError()

    def tok_encode(self, string: str):
        return self.tokenizer.encode(string, add_special_tokens=False)

    def tok_decode(self, tokens):
        return self.tokenizer.decode(tokens)