(local_rank, dataset_name, output_path,
seed, add_sys_prefix=False, for_backbone=False, sample_ratio=None)
| 263 | |
| 264 | # step 2 |
| 265 | def create_dataset(local_rank, dataset_name, output_path, |
| 266 | seed, add_sys_prefix=False, for_backbone=False, sample_ratio=None): |
| 267 | raw_dataset = get_raw_dataset(dataset_name, output_path, seed, local_rank, for_backbone=for_backbone) |
| 268 | |
| 269 | train_dataset = raw_dataset.get_train_data() |
| 270 | train_dataset = get_prompt_dataset(train_dataset, raw_dataset, add_sys_prefix=add_sys_prefix, sample_ratio=sample_ratio) |
| 271 | |
| 272 | eval_dataset = raw_dataset.get_eval_data() |
| 273 | eval_dataset = get_prompt_dataset(eval_dataset, raw_dataset, add_sys_prefix=add_sys_prefix) |
| 274 | |
| 275 | test_dataset = raw_dataset.get_test_data() |
| 276 | test_dataset = get_prompt_dataset(test_dataset, raw_dataset, add_sys_prefix=add_sys_prefix) |
| 277 | |
| 278 | return train_dataset, eval_dataset, test_dataset |
| 279 | |
| 280 | |
| 281 | # step 1 |
no test coverage detected