Method __call__

Megatron-LM/data_utils/tokenization.py:245–247

run preprocessing and encode text as Ids

(self, text, process_fn=None)

    def __call__(self, text, process_fn=None):
        """run preprocessing and encode text as Ids"""
        return self.EncodeAsIds(text, process_fn=process_fn)

    def __len__(self):
        """total number of tokens"""