Method tokenize

DeepSpeedExample/tools/preprocess_data.py:52–53

(self, *text)

class IdentitySplitter(object):
    def tokenize(self, *text):
        return text

class Encoder(object):
    def __init__(self, args):