Treat a list of paths as files belonging to a single file system If the file system is local then also validates that all paths are referencing existing *files* otherwise any non-file paths will be silently skipped (for example on a remote filesystem). Parameters ---------
(paths, filesystem=None)
| 318 | |
| 319 | |
| 320 | def _ensure_multiple_sources(paths, filesystem=None): |
| 321 | """ |
| 322 | Treat a list of paths as files belonging to a single file system |
| 323 | |
| 324 | If the file system is local then also validates that all paths |
| 325 | are referencing existing *files* otherwise any non-file paths will be |
| 326 | silently skipped (for example on a remote filesystem). |
| 327 | |
| 328 | Parameters |
| 329 | ---------- |
| 330 | paths : list of path-like |
| 331 | Note that URIs are not allowed. |
| 332 | filesystem : FileSystem or str, optional |
| 333 | If an URI is passed, then its path component will act as a prefix for |
| 334 | the file paths. |
| 335 | |
| 336 | Returns |
| 337 | ------- |
| 338 | (FileSystem, list of str) |
| 339 | File system object and a list of normalized paths. |
| 340 | |
| 341 | Raises |
| 342 | ------ |
| 343 | TypeError |
| 344 | If the passed filesystem has wrong type. |
| 345 | IOError |
| 346 | If the file system is local and a referenced path is not available or |
| 347 | not a file. |
| 348 | """ |
| 349 | from pyarrow.fs import ( |
| 350 | LocalFileSystem, SubTreeFileSystem, _MockFileSystem, FileType, |
| 351 | _ensure_filesystem |
| 352 | ) |
| 353 | |
| 354 | if filesystem is None: |
| 355 | # fall back to local file system as the default |
| 356 | filesystem = LocalFileSystem() |
| 357 | else: |
| 358 | # construct a filesystem if it is a valid URI |
| 359 | filesystem = _ensure_filesystem(filesystem) |
| 360 | |
| 361 | is_local = ( |
| 362 | isinstance(filesystem, (LocalFileSystem, _MockFileSystem)) or |
| 363 | (isinstance(filesystem, SubTreeFileSystem) and |
| 364 | isinstance(filesystem.base_fs, LocalFileSystem)) |
| 365 | ) |
| 366 | |
| 367 | # allow normalizing irregular paths such as Windows local paths |
| 368 | paths = [filesystem.normalize_path(_stringify_path(p)) for p in paths] |
| 369 | |
| 370 | # validate that all of the paths are pointing to existing *files* |
| 371 | # possible improvement is to group the file_infos by type and raise for |
| 372 | # multiple paths per error category |
| 373 | if is_local: |
| 374 | for info in filesystem.get_file_info(paths): |
| 375 | file_type = info.type |
| 376 | if file_type == FileType.File: |
| 377 | continue |
no test coverage detected