return dataset id when getting data from ModelScope/HuggingFace repo, otherwise just return local path as is. Args: dataset_id (str): dataset id or data path local_mode (bool): whether to use local path or ModelScope/HuggignFace repo
(dataset_id: str, local_mode: bool = False)
| 8 | |
| 9 | |
| 10 | def get_data_path(dataset_id: str, local_mode: bool = False): |
| 11 | """return dataset id when getting data from ModelScope/HuggingFace repo, otherwise just |
| 12 | return local path as is. |
| 13 | |
| 14 | Args: |
| 15 | dataset_id (str): dataset id or data path |
| 16 | local_mode (bool): whether to use local path or |
| 17 | ModelScope/HuggignFace repo |
| 18 | """ |
| 19 | # update the path with CACHE_DIR |
| 20 | cache_dir = os.environ.get('COMPASS_DATA_CACHE', '') |
| 21 | |
| 22 | # For absolute path customized by the users |
| 23 | if dataset_id.startswith('/'): |
| 24 | return dataset_id |
| 25 | |
| 26 | # For relative path, with CACHE_DIR |
| 27 | if local_mode: |
| 28 | local_path = os.path.join(cache_dir, dataset_id) |
| 29 | if not os.path.exists(local_path): |
| 30 | return download_dataset(local_path, cache_dir) |
| 31 | else: |
| 32 | return local_path |
| 33 | |
| 34 | dataset_source = os.environ.get('DATASET_SOURCE', None) |
| 35 | if dataset_source == 'ModelScope': |
| 36 | ms_id = DATASETS_MAPPING[dataset_id]['ms_id'] |
| 37 | assert ms_id is not None, \ |
| 38 | f'{dataset_id} is not supported in ModelScope' |
| 39 | return ms_id |
| 40 | elif dataset_source == 'HF': |
| 41 | # TODO: HuggingFace mode is currently not supported! |
| 42 | hf_id = DATASETS_MAPPING[dataset_id]['hf_id'] |
| 43 | assert hf_id is not None, \ |
| 44 | f'{dataset_id} is not supported in HuggingFace' |
| 45 | return hf_id |
| 46 | else: |
| 47 | # for the local path |
| 48 | local_path = DATASETS_MAPPING[dataset_id]['local'] |
| 49 | local_path = os.path.join(cache_dir, local_path) |
| 50 | |
| 51 | if not os.path.exists(local_path): |
| 52 | return download_dataset(local_path, cache_dir) |
| 53 | else: |
| 54 | return local_path |
| 55 | |
| 56 | |
| 57 | def download_dataset(data_path, cache_dir, remove_finished=True): |