Method __call__

data_utils/tokenization.py:269–271

run preprocessing and encode text as Ids

(self, text, process_fn=None)

        self._token_type_vocab = {t: Id for Id, t in self.type_id_map.items()}

    def __call__(self, text, process_fn=None):
        """run preprocessing and encode text as Ids"""
        return self.EncodeAsIds(text, process_fn=process_fn)

    def __len__(self):
        """total number of tokens"""