Train/valid/test data arguments.
(parser)
| 225 | |
| 226 | |
| 227 | def add_data_args(parser): |
| 228 | """Train/valid/test data arguments.""" |
| 229 | |
| 230 | group = parser.add_argument_group('data', 'data configurations') |
| 231 | |
| 232 | # Training datasets. |
| 233 | group.add_argument('--train-data', nargs='+', default=None, |
| 234 | help='Whitespace separated filenames or corpora names for training.' |
| 235 | 'Use hf://path/to/dataset to load huggingface datasets.') |
| 236 | group.add_argument('--train-data-weights', nargs='+', default=None, type=int, |
| 237 | help='scaling factors for different train-data, must the same number of' |
| 238 | '--train-data or None(==1).') |
| 239 | parser.add_argument('--iterable-dataset', |
| 240 | action='store', default=False, const=True, nargs='?', choices=[True, False, 'custom'], help='custom means Dataloader batch_size=None, and you need to generate batches by yourself. This is useful for dynamic batchsize.') |
| 241 | parser.add_argument('--iterable-dataset-eval', default='', |
| 242 | type=str, choices=['True', 'False', ''], help='Whether eval is iterable.') |
| 243 | |
| 244 | group.add_argument('--batch-from-same-dataset', action='store_true', |
| 245 | help='batch from the same dataset, not random. ONLY affect len(train-data-weights) > 1 and iterable dataset') |
| 246 | |
| 247 | # Validation and Test dataset. |
| 248 | group.add_argument('--valid-data', nargs='*', default=None, |
| 249 | help="""Filename for validation data.""") |
| 250 | group.add_argument('--test-data', nargs='*', default=None, |
| 251 | help="""Filename for testing""") |
| 252 | group.add_argument('--split', default='1000,1,1', |
| 253 | help='comma-separated list of proportions for training,' |
| 254 | ' validation, and test split. ' |
| 255 | 'Only take effects when no valid/test specified.') |
| 256 | |
| 257 | # Efficiency. |
| 258 | group.add_argument('--num-workers', type=int, default=1, |
| 259 | help="""Number of workers to use for dataloading""") |
| 260 | # Sometimes, num-workders > 1 and cpu_offload (zero-stage 2) will make the validation dataloader hang. |
| 261 | # I have no idea on the reason, and just set the default num-workers to 1. |
| 262 | group.add_argument('--block-size', type=int, default=10000, |
| 263 | help="""Size of block to reduce memory in dataset, ignore it for most users.""") |
| 264 | group.add_argument('--prefetch-factor', type=int, default=4, |
| 265 | help='prefetching number of batches.') |
| 266 | return parser |
| 267 | |
| 268 | |
| 269 | def add_text_generate_args(parser): |