Method encode

utils/tokenizer.py:53–56

(self, text: str)

        self.chars_per_token = chars_per_token

    def encode(self, text: str) -> List[int]:
        # Return pseudo list of estimated token count
        estimated_count = int(len(text) / self.chars_per_token)
        return list(range(estimated_count))

    def decode(self, tokens: List[int]) -> str:
        # Cannot truly decode, return empty string