| 78 | |
| 79 | |
| 80 | class LocalJsonFileDataset(PromptRawDataset): |
| 81 | |
| 82 | def __init__(self, output_path, seed, local_rank, dataset_name, for_backbone=False): |
| 83 | super().__init__(output_path, seed, local_rank, dataset_name) |
| 84 | self.dataset_name = "local_jsonfile" |
| 85 | self.dataset_name_clean = "jsonfile" |
| 86 | assert os.path.exists(dataset_name), f"Not found, plz check path {dataset_name}!" |
| 87 | self.for_backbone = for_backbone |
| 88 | self.raw_datasets = load_dataset('json', |
| 89 | data_files={ |
| 90 | "train": |
| 91 | dataset_name + '/train.json', |
| 92 | "eval": |
| 93 | dataset_name + '/eval.json', |
| 94 | "test": |
| 95 | dataset_name + '/test.json', |
| 96 | }) |
| 97 | |
| 98 | def get_train_data(self): |
| 99 | if self.raw_datasets['train'] is not None: |
| 100 | return self.raw_datasets['train'] |
| 101 | return None |
| 102 | |
| 103 | def get_eval_data(self): |
| 104 | if self.raw_datasets['eval'] is not None: |
| 105 | return self.raw_datasets['eval'] |
| 106 | return None |
| 107 | |
| 108 | def get_test_data(self): |
| 109 | if self.raw_datasets['test'] is not None: |
| 110 | return self.raw_datasets['test'] |
| 111 | return None |
| 112 | |
| 113 | def get_prompt(self, sample): |
| 114 | if sample['prompt'] is not None: |
| 115 | return sample['prompt'] |
| 116 | return None |
| 117 | |
| 118 | def get_answer(self, sample): |
| 119 | if sample['answer'] is not None: |
| 120 | return sample['answer'] |
| 121 | return '' |
| 122 | |
| 123 | def get_prompt_and_answer(self, sample): |
| 124 | if sample['prompt'] is not None and sample['answer'] is not None: |
| 125 | return sample['prompt'] + "\n" + sample['answer'] |
| 126 | return None |
nothing calls this directly
no outgoing calls
no test coverage detected