Return: meta_map: Structure of meta files (.csv), the meta file name will be replaced by url, like { "test": "https://xxx/mytest.csv" } file_map: Structure of data files (.zip), like { "test": "pictures.zip" }
(subset_split_into: dict,
dataset_name: str,
namespace: str,
context_config: DatasetContextConfig,
revision: Optional[str] = DEFAULT_DATASET_REVISION)
| 171 | |
| 172 | |
| 173 | def get_dataset_files(subset_split_into: dict, |
| 174 | dataset_name: str, |
| 175 | namespace: str, |
| 176 | context_config: DatasetContextConfig, |
| 177 | revision: Optional[str] = DEFAULT_DATASET_REVISION): |
| 178 | """ |
| 179 | Return: |
| 180 | meta_map: Structure of meta files (.csv), the meta file name will be replaced by url, like |
| 181 | { |
| 182 | "test": "https://xxx/mytest.csv" |
| 183 | } |
| 184 | file_map: Structure of data files (.zip), like |
| 185 | { |
| 186 | "test": "pictures.zip" |
| 187 | } |
| 188 | """ |
| 189 | meta_map = defaultdict(dict) |
| 190 | file_map = defaultdict(dict) |
| 191 | args_map = defaultdict(dict) |
| 192 | custom_type_map = defaultdict(dict) |
| 193 | modelscope_api = HubApi() |
| 194 | meta_cache_dir = context_config.data_meta_config.meta_cache_dir |
| 195 | |
| 196 | for split, info in subset_split_into.items(): |
| 197 | custom_type_map[split] = info.get('custom', '') |
| 198 | meta_map[split] = modelscope_api.get_dataset_file_url_origin( |
| 199 | info.get('meta', ''), dataset_name, namespace, revision) |
| 200 | if info.get('file'): |
| 201 | file_map[split] = info['file'] |
| 202 | args_map[split] = info.get('args') |
| 203 | |
| 204 | objects = [] |
| 205 | # If `big_data` is true, then fetch objects from meta-csv file directly. |
| 206 | for split, args_dict in args_map.items(): |
| 207 | if args_dict and args_dict.get(MetaDataFields.ARGS_BIG_DATA): |
| 208 | meta_csv_file_url = meta_map[split] |
| 209 | |
| 210 | meta_csv_file_path = HubApi.fetch_meta_files_from_url( |
| 211 | meta_csv_file_url, meta_cache_dir) |
| 212 | |
| 213 | csv_delimiter = context_config.config_kwargs.get('delimiter', ',') |
| 214 | csv_df = pd.read_csv( |
| 215 | meta_csv_file_path, |
| 216 | iterator=False, |
| 217 | delimiter=csv_delimiter, |
| 218 | escapechar='\\') |
| 219 | target_col = csv_df.columns[csv_df.columns.str.contains( |
| 220 | ':FILE')].to_list() |
| 221 | if len(target_col) == 0: |
| 222 | logger.error( |
| 223 | f'No column contains ":FILE" in {meta_csv_file_path}.') |
| 224 | target_col = csv_df.columns[0] |
| 225 | else: |
| 226 | target_col = target_col[0] |
| 227 | objects = csv_df[target_col].to_list() |
| 228 | |
| 229 | file_map[split] = objects |
| 230 | # More general but low-efficiency. |
no test coverage detected
searching dependent graphs…