Resolve data file paths/URLs from a user-supplied pattern. Supports ``*``, ``**``, and fsspec-based remote patterns (e.g. ``hf://``). Hidden files/directories and ``__pycache__`` are excluded by default.
(
pattern: str,
base_path: str,
allowed_extensions: Optional[List[str]] = None,
download_config: Optional[DownloadConfig] = None,
)
| 176 | |
| 177 | |
| 178 | def _resolve_pattern( |
| 179 | pattern: str, |
| 180 | base_path: str, |
| 181 | allowed_extensions: Optional[List[str]] = None, |
| 182 | download_config: Optional[DownloadConfig] = None, |
| 183 | ) -> List[str]: |
| 184 | """Resolve data file paths/URLs from a user-supplied pattern. |
| 185 | |
| 186 | Supports ``*``, ``**``, and fsspec-based remote patterns (e.g. ``hf://``). |
| 187 | Hidden files/directories and ``__pycache__`` are excluded by default. |
| 188 | """ |
| 189 | if is_relative_path(pattern): |
| 190 | pattern = xjoin(base_path, pattern) |
| 191 | elif is_local_path(pattern): |
| 192 | base_path = os.path.splitdrive(pattern)[0] + os.sep |
| 193 | else: |
| 194 | base_path = '' |
| 195 | pattern, storage_options = _prepare_path_and_storage_options( |
| 196 | pattern, download_config=download_config) |
| 197 | fs = get_fs_token_paths(pattern, storage_options=storage_options) |
| 198 | fs_base_path = base_path.split('::')[0].split('://')[-1] or fs.root_marker |
| 199 | fs_pattern = pattern.split('::')[0].split('://')[-1] |
| 200 | files_to_ignore = set(FILES_TO_IGNORE) - {xbasename(pattern)} |
| 201 | protocol = fs.protocol if isinstance(fs.protocol, str) else fs.protocol[0] |
| 202 | protocol_prefix = protocol + '://' if protocol != 'file' else '' |
| 203 | glob_kwargs = {} |
| 204 | if protocol == 'hf' and config.HF_HUB_VERSION >= version.parse('0.20.0'): |
| 205 | glob_kwargs['expand_info'] = False |
| 206 | |
| 207 | try: |
| 208 | tmp_file_paths = fs.glob(pattern, detail=True, **glob_kwargs) |
| 209 | except FileNotFoundError: |
| 210 | raise DataFilesNotFoundError(f"Unable to find '{pattern}'") |
| 211 | |
| 212 | matched_paths = [ |
| 213 | filepath if filepath.startswith(protocol_prefix) else protocol_prefix |
| 214 | + filepath for filepath, info in tmp_file_paths.items() |
| 215 | if info['type'] == 'file' and ( |
| 216 | xbasename(filepath) not in files_to_ignore) |
| 217 | and not _is_inside_unrequested_special_dir( |
| 218 | os.path.relpath(filepath, fs_base_path), |
| 219 | os.path.relpath(fs_pattern, fs_base_path)) and # noqa: W504 |
| 220 | not _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir( # noqa: W504 |
| 221 | os.path.relpath(filepath, fs_base_path), |
| 222 | os.path.relpath(fs_pattern, fs_base_path)) |
| 223 | ] |
| 224 | if allowed_extensions is not None: |
| 225 | out = [ |
| 226 | filepath for filepath in matched_paths |
| 227 | if any('.' + suffix in allowed_extensions |
| 228 | for suffix in xbasename(filepath).split('.')[1:]) |
| 229 | ] |
| 230 | if len(out) < len(matched_paths): |
| 231 | invalid_matched_files = list(set(matched_paths) - set(out)) |
| 232 | logger.info( |
| 233 | f"Some files matched the pattern '{pattern}' but don't have valid data file extensions: " |
| 234 | f'{invalid_matched_files}') |
| 235 | else: |
nothing calls this directly
no test coverage detected
searching dependent graphs…