Method tokenizer

data_utils/datasets.py:310–313

(self)

    @property
    def tokenizer(self):
        if self.using_tokenizer:
            return self._tokenizer
        return None

    def __len__(self):
        return len(self.X)