(path)
| 154 | |
| 155 | @staticmethod |
| 156 | def load(path): |
| 157 | path = get_data_path(path) |
| 158 | if environ.get('DATASET_SOURCE') == 'ModelScope': |
| 159 | from modelscope import MsDataset |
| 160 | dataset = DatasetDict() |
| 161 | for split in ['train', 'validation']: |
| 162 | data_list = [] |
| 163 | ms_dataset = MsDataset.load(path, split=split) |
| 164 | for item in ms_dataset: |
| 165 | row = item |
| 166 | label = item['label'] |
| 167 | # some preprocessing |
| 168 | row['A'] = item['answerA'] |
| 169 | row['B'] = item['answerB'] |
| 170 | row['C'] = item['answerC'] |
| 171 | row['answer'] = 'ABC'[int(label) - 1] |
| 172 | del row['answerA'], row['answerB'], row['answerC'], row[ |
| 173 | 'label'] |
| 174 | data_list.append(row) |
| 175 | dataset[split] = Dataset.from_list(data_list) |
| 176 | else: |
| 177 | train_dataset = SiqaDatasetV3.load_single(path, 'train.jsonl', |
| 178 | 'train-labels.lst') |
| 179 | val_dataset = SiqaDatasetV3.load_single(path, 'dev.jsonl', |
| 180 | 'dev-labels.lst') |
| 181 | dataset = DatasetDict({ |
| 182 | 'train': train_dataset, |
| 183 | 'validation': val_dataset |
| 184 | }) |
| 185 | return dataset |
nothing calls this directly
no test coverage detected