logic to set and remove (set to None) tokenizer. combines preprocessing/tokenization into one callable.
(self, tokenizer)
| 182 | self.is_lazy = True |
| 183 | |
| 184 | def SetTokenizer(self, tokenizer): |
| 185 | """ |
| 186 | logic to set and remove (set to None) tokenizer. |
| 187 | combines preprocessing/tokenization into one callable. |
| 188 | """ |
| 189 | if tokenizer is None: |
| 190 | if not hasattr(self, '_tokenizer'): |
| 191 | self._tokenizer = tokenizer |
| 192 | else: |
| 193 | self._tokenizer = tokenizer |
| 194 | self.map_fn = ProcessorTokenizer(tokenizer, self.process_fn) |
| 195 | |
| 196 | def GetTokenizer(self): |
| 197 | return self._tokenizer |
nothing calls this directly
no test coverage detected