Method __init__

data_utils/tokenization.py:551–555

(self, **kwargs)

    """

    def __init__(self, **kwargs):
        self.num_text_tokens = 256
        super(CharacterLevelTokenizer, self).__init__()
        self._tokens = [self.IdToToken(Id) for Id in range(self.num_text_tokens)]
        self._vocab = {t: i for i, t in enumerate(self._tokens)}

    def __len__(self):
        return 256