makes training/val/test Args: args.train_data, args.valid_data, args.test_data: str. Paths to the dataset. args.split: str. format: "8,1,1". how to split train_data. args.dataset_type: use to create the right datasets.
(args, create_dataset_function, collate_fn=None)
| 169 | return train_ds, valid_ds, test_ds |
| 170 | |
| 171 | def make_loaders(args, create_dataset_function, collate_fn=None): |
| 172 | """makes training/val/test |
| 173 | Args: |
| 174 | args.train_data, args.valid_data, args.test_data: str. Paths to the dataset. |
| 175 | args.split: str. format: "8,1,1". how to split train_data. |
| 176 | args.dataset_type: use to create the right datasets. |
| 177 | """ |
| 178 | make_dataset = partial(make_dataset_full, |
| 179 | create_dataset_function=create_dataset_function, batch_from_same_dataset=args.batch_from_same_dataset) |
| 180 | |
| 181 | world_size = torch.distributed.get_world_size( |
| 182 | group=mpu.get_data_parallel_group()) |
| 183 | batch_size = args.batch_size * world_size |
| 184 | eval_batch_size = batch_size |
| 185 | if args.eval_batch_size is not None: |
| 186 | eval_batch_size = args.eval_batch_size * world_size |
| 187 | |
| 188 | split = get_split(args) |
| 189 | |
| 190 | data_set_args = { |
| 191 | 'path': args.train_data, # * here the path is used. |
| 192 | 'split': split, |
| 193 | } |
| 194 | |
| 195 | eval_set_args = copy.copy(data_set_args) |
| 196 | eval_set_args['split'] = [1.] |
| 197 | |
| 198 | # make datasets splits and tokenizer |
| 199 | train = None |
| 200 | valid = None |
| 201 | test = None |
| 202 | |
| 203 | if args.train_data is not None: |
| 204 | train = make_dataset(**data_set_args, args=args, dataset_weights=args.train_data_weights, is_train_data=True) |
| 205 | if should_split(split): |
| 206 | train, valid, test = train |
| 207 | |
| 208 | # make training and val dataset if necessary |
| 209 | if valid is None and args.valid_data is not None: |
| 210 | eval_set_args['path'] = args.valid_data |
| 211 | valid = make_dataset(**eval_set_args, args=args, random_mapping=not args.strict_eval) |
| 212 | if test is None and args.test_data is not None: |
| 213 | eval_set_args['path'] = args.test_data |
| 214 | test = make_dataset(**eval_set_args, args=args, random_mapping=not args.strict_eval) |
| 215 | |
| 216 | # wrap datasets with data loader |
| 217 | if train is not None and args.batch_size > 0: |
| 218 | train = make_data_loader(train, batch_size, args, split='train', collate_fn=collate_fn) |
| 219 | args.do_train = True |
| 220 | else: |
| 221 | args.do_train = False |
| 222 | eval_batch_size = eval_batch_size if eval_batch_size != 0 else batch_size |
| 223 | if valid is not None: |
| 224 | valid = make_data_loader(valid, eval_batch_size, args, split='val', collate_fn=collate_fn) |
| 225 | args.do_valid = True |
| 226 | else: |
| 227 | args.do_valid = False |
| 228 | if test is not None: |
no test coverage detected