| 11 | from seq2struct.utils import vocab |
| 12 | |
| 13 | class Preprocessor: |
| 14 | def __init__(self, config): |
| 15 | self.config = config |
| 16 | self.model_preproc = registry.instantiate( |
| 17 | registry.lookup('model', config['model']).Preproc, |
| 18 | config['model']) |
| 19 | |
| 20 | def preprocess(self): |
| 21 | self.model_preproc.clear_items() |
| 22 | for section in self.config['data']: |
| 23 | data = registry.construct('dataset', self.config['data'][section]) |
| 24 | for item in tqdm.tqdm(data, desc=section, dynamic_ncols=True): |
| 25 | to_add, validation_info = self.model_preproc.validate_item(item, section) |
| 26 | if to_add: |
| 27 | self.model_preproc.add_item(item, section, validation_info) |
| 28 | self.model_preproc.save() |
| 29 | |
| 30 | def add_parser(): |
| 31 | parser = argparse.ArgumentParser() |