| 246 | |
| 247 | @staticmethod |
| 248 | def get_default_config(updates=None): |
| 249 | config = ConfigDict() |
| 250 | config.path = 'c4' |
| 251 | config.name = 'en' |
| 252 | config.split = 'train' |
| 253 | config.streaming = False |
| 254 | config.seq_length = 1024 |
| 255 | config.batch_size = 8 |
| 256 | config.always_start_with_bos = False |
| 257 | |
| 258 | if updates is not None: |
| 259 | config.update(ConfigDict(updates).copy_and_resolve_references()) |
| 260 | return config |
| 261 | |
| 262 | def __init__(self, config, tokenizer, text_processor): |
| 263 | self.config = self.get_default_config(config) |