add cmdline flags for configuring datasets
()
| 399 | |
| 400 | |
| 401 | def configure_data(): |
| 402 | """add cmdline flags for configuring datasets""" |
| 403 | # These are options that are used by data_utils, but are either |
| 404 | # deprecated or not meant to be exposed to the command line user. |
| 405 | # These options are intneded to be set in code by specific scripts. |
| 406 | defaults = { |
| 407 | 'world_size': 1, |
| 408 | 'rank': -1, |
| 409 | 'persist_state': 0, |
| 410 | 'lazy': False, |
| 411 | 'transpose': False, |
| 412 | 'data_set_type': 'supervised', |
| 413 | 'seq_length': 256, |
| 414 | 'eval_seq_length': 256, |
| 415 | 'samples_per_shard': 100 |
| 416 | } |
| 417 | |
| 418 | return DataConfig(defaults=defaults) |
no test coverage detected