(path)
| 33 | |
| 34 | @staticmethod |
| 35 | def load(path): |
| 36 | path = get_data_path(path) |
| 37 | if environ.get('DATASET_SOURCE') == 'ModelScope': |
| 38 | from modelscope import MsDataset |
| 39 | dataset = DatasetDict() |
| 40 | for split in ['train', 'validation']: |
| 41 | data_list = [] |
| 42 | ms_dataset = MsDataset.load(path, split=split) |
| 43 | for item in ms_dataset: |
| 44 | row = item |
| 45 | row['label'] = int(item['label']) |
| 46 | data_list.append(row) |
| 47 | dataset[split] = Dataset.from_list(data_list) |
| 48 | return dataset |
| 49 | else: |
| 50 | train_dataset = siqaDataset.load_single(path, 'train.jsonl', |
| 51 | 'train-labels.lst') |
| 52 | val_dataset = siqaDataset.load_single(path, 'dev.jsonl', |
| 53 | 'dev-labels.lst') |
| 54 | return DatasetDict({ |
| 55 | 'train': train_dataset, |
| 56 | 'validation': val_dataset |
| 57 | }) |
| 58 | |
| 59 | |
| 60 | @LOAD_DATASET.register_module() |
no test coverage detected