Class FakeTokenizer

SwissArmyTransformer/sat/tokenization/__init__.py:86–91

class FakeTokenizer(object):
    def __init__(self, num_tokens):
        self.num_tokens = num_tokens

    def __len__(self):
        return self.num_tokens