Method Train

data_utils/tokenization.py:511–513

train a tokenizer on a data corpus and save model for future use

(self, corpus)

        raise NotImplementedError('TextTokenizer exists method not implemented')

    def Train(self, corpus):
        """train a tokenizer on a data corpus and save model for future use"""
        raise NotImplementedError('TextTokenizer Train not implemented')

    def EncodeAsIds(self, text, process_fn=None):
        """