Method encode

utils/tokenizer.py:40–41

(self, text: str)

        self.encoding_name = encoding_name

    def encode(self, text: str) -> List[int]:
        return self._encoding.encode(text)

    def decode(self, tokens: List[int]) -> str:
        return self._encoding.decode(tokens)