(path: str, local_mode: bool = False)
| 27 | |
| 28 | @staticmethod |
| 29 | def load(path: str, local_mode: bool = False): |
| 30 | path = get_data_path(path, local_mode=local_mode) |
| 31 | |
| 32 | def processing_test(example): |
| 33 | example['test_case'] = example['test_list'] |
| 34 | example['test_list'] = '\n'.join(example['test_list']) |
| 35 | example['test_list_2'] = example['test_list'] |
| 36 | return example |
| 37 | |
| 38 | if environ.get('DATASET_SOURCE') == 'ModelScope': |
| 39 | from modelscope import MsDataset |
| 40 | train = MsDataset.load(path, |
| 41 | subset_name='full', |
| 42 | split='train[:10]').map(processing_test) |
| 43 | test = MsDataset.load(path, |
| 44 | subset_name='full', |
| 45 | split='train[10:510]').map(processing_test) |
| 46 | else: |
| 47 | train = load_dataset('json', data_files=path, |
| 48 | split='train[:10]').map(processing_test) |
| 49 | test = load_dataset('json', data_files=path, |
| 50 | split='train[10:510]').map(processing_test) |
| 51 | return DatasetDict({'train': train, 'test': test}) |
| 52 | |
| 53 | |
| 54 | class MBPPDatasetV2(BaseDataset): |
no test coverage detected