callable class that runs a preprocessing, as well as tokenization step, on input text.
| 102 | |
| 103 | |
| 104 | class ProcessorTokenizer: |
| 105 | """ |
| 106 | callable class that runs a preprocessing, as well as tokenization step, |
| 107 | on input text. |
| 108 | """ |
| 109 | |
| 110 | def __init__(self, tokenizer, process_fn=None): |
| 111 | self.tokenizer = tokenizer |
| 112 | self.process_fn = process_fn |
| 113 | |
| 114 | def __call__(self, string): |
| 115 | if self.tokenizer is not None: |
| 116 | string = self.tokenizer(string, process_fn=self.process_fn) |
| 117 | elif self.process_fn is not None: |
| 118 | string = self.process_fn(string) |
| 119 | return string |
| 120 | |
| 121 | |
| 122 | class LazyLoader(object): |