| 93 | |
| 94 | |
| 95 | class DataConfig: |
| 96 | |
| 97 | def __init__(self, defaults=None): |
| 98 | super(DataConfig, self).__init__() |
| 99 | if defaults is None: |
| 100 | defaults = {} |
| 101 | self.defaults = defaults |
| 102 | |
| 103 | def apply(self, args, tokenizer): |
| 104 | if torch.distributed.get_rank() == 0: |
| 105 | print('configuring data') |
| 106 | self.apply_defaults(args) |
| 107 | return make_loaders(args, tokenizer) |
| 108 | |
| 109 | def set_defaults(self, **kwargs): |
| 110 | for k, v in kwargs.items(): |
| 111 | self.defaults[k] = v |
| 112 | |
| 113 | def apply_defaults(self, args): |
| 114 | for k, v in self.defaults.items(): |
| 115 | k = k.replace('-', '_') |
| 116 | if not hasattr(args, k): |
| 117 | setattr(args, k, v) |
| 118 | |
| 119 | |
| 120 | def prepare_tokenizer(args): |