Method EncodeAsIds

data_utils/tokenization.py:515–520

Preprocess text and encode as ids. Return a tokenization object with original text, processed text, and id tokenization.

(self, text, process_fn=None)

        raise NotImplementedError('TextTokenizer Train not implemented')

    def EncodeAsIds(self, text, process_fn=None):
        """
        Preprocess text and encode as ids. Return a tokenization object with
        original text, processed text, and id tokenization.
        """
        raise NotImplementedError('TextTokenizer EncodeAsIds not implemented')

    def EncodeAsTokens(self, text, process_fn=None):
        """