Train/valid/test data arguments.
(parser)
| 218 | |
| 219 | |
| 220 | def add_data_args(parser): |
| 221 | """Train/valid/test data arguments.""" |
| 222 | |
| 223 | group = parser.add_argument_group('data', 'data configurations') |
| 224 | |
| 225 | group.add_argument('--model-parallel-size', type=int, default=1, |
| 226 | help='size of the model parallel.') |
| 227 | group.add_argument('--shuffle', action='store_true', |
| 228 | help='Shuffle data. Shuffling is deterministic ' |
| 229 | 'based on seed and current epoch.') |
| 230 | group.add_argument('--train-data', nargs='+', default=None, |
| 231 | help='Whitespace separated filenames or corpora names ' |
| 232 | 'for training.') |
| 233 | |
| 234 | group.add_argument('--use-npy-data-loader', action='store_true', |
| 235 | help='Use the numpy data loader. If set, then' |
| 236 | 'train-data-path, val-data-path, and test-data-path' |
| 237 | 'should also be provided.') |
| 238 | group.add_argument('--train-data-path', type=str, default='', |
| 239 | help='path to the training data') |
| 240 | group.add_argument('--val-data-path', type=str, default='', |
| 241 | help='path to the validation data') |
| 242 | group.add_argument('--test-data-path', type=str, default='', |
| 243 | help='path to the test data') |
| 244 | group.add_argument('--input-data-sizes-file', type=str, default='sizes.txt', |
| 245 | help='the filename containing all the shards sizes') |
| 246 | |
| 247 | group.add_argument('--delim', default=',', |
| 248 | help='delimiter used to parse csv data files') |
| 249 | group.add_argument('--text-key', default='sentence', |
| 250 | help='key to use to extract text from json/csv') |
| 251 | group.add_argument('--eval-text-key', default=None, |
| 252 | help='key to use to extract text from ' |
| 253 | 'json/csv evaluation datasets') |
| 254 | group.add_argument('--valid-data', nargs='*', default=None, |
| 255 | help="""Filename for validation data.""") |
| 256 | group.add_argument('--split', default='1000,1,1', |
| 257 | help='comma-separated list of proportions for training,' |
| 258 | ' validation, and test split') |
| 259 | group.add_argument('--test-data', nargs='*', default=None, |
| 260 | help="""Filename for testing""") |
| 261 | |
| 262 | group.add_argument('--lazy-loader', action='store_true', |
| 263 | help='whether to lazy read the data set') |
| 264 | group.add_argument('--loose-json', action='store_true', |
| 265 | help='Use loose json (one json-formatted string per ' |
| 266 | 'newline), instead of tight json (data file is one ' |
| 267 | 'json string)') |
| 268 | group.add_argument('--presplit-sentences', action='store_true', |
| 269 | help='Dataset content consists of documents where ' |
| 270 | 'each document consists of newline separated sentences') |
| 271 | group.add_argument('--num-workers', type=int, default=2, |
| 272 | help="""Number of workers to use for dataloading""") |
| 273 | group.add_argument('--tokenizer-model-type', type=str, |
| 274 | default='bert-large-uncased', |
| 275 | help="Model type to use for sentencepiece tokenization \ |
| 276 | (one of ['bpe', 'char', 'unigram', 'word']) or \ |
| 277 | bert vocab to use for BertWordPieceTokenizer (one of \ |