Create a FileSystemDataset which can be used to build a Dataset. Parameters are documented in the dataset function. Returns ------- FileSystemDataset
(source, schema=None, filesystem=None,
partitioning=None, format=None,
partition_base_dir=None, exclude_invalid_files=None,
selector_ignore_prefixes=None)
| 440 | |
| 441 | |
| 442 | def _filesystem_dataset(source, schema=None, filesystem=None, |
| 443 | partitioning=None, format=None, |
| 444 | partition_base_dir=None, exclude_invalid_files=None, |
| 445 | selector_ignore_prefixes=None): |
| 446 | """ |
| 447 | Create a FileSystemDataset which can be used to build a Dataset. |
| 448 | |
| 449 | Parameters are documented in the dataset function. |
| 450 | |
| 451 | Returns |
| 452 | ------- |
| 453 | FileSystemDataset |
| 454 | """ |
| 455 | from pyarrow.fs import LocalFileSystem, _ensure_filesystem, FileInfo |
| 456 | |
| 457 | format = _ensure_format(format or 'parquet') |
| 458 | partitioning = _ensure_partitioning(partitioning) |
| 459 | |
| 460 | if isinstance(source, (list, tuple)): |
| 461 | if source and isinstance(source[0], FileInfo): |
| 462 | if filesystem is None: |
| 463 | # fall back to local file system as the default |
| 464 | fs = LocalFileSystem() |
| 465 | else: |
| 466 | # construct a filesystem if it is a valid URI |
| 467 | fs = _ensure_filesystem(filesystem) |
| 468 | paths_or_selector = source |
| 469 | else: |
| 470 | fs, paths_or_selector = _ensure_multiple_sources(source, filesystem) |
| 471 | else: |
| 472 | fs, paths_or_selector = _ensure_single_source(source, filesystem) |
| 473 | |
| 474 | options = FileSystemFactoryOptions( |
| 475 | partitioning=partitioning, |
| 476 | partition_base_dir=partition_base_dir, |
| 477 | exclude_invalid_files=exclude_invalid_files, |
| 478 | selector_ignore_prefixes=selector_ignore_prefixes |
| 479 | ) |
| 480 | factory = FileSystemDatasetFactory(fs, paths_or_selector, format, options) |
| 481 | |
| 482 | return factory.finish(schema) |
| 483 | |
| 484 | |
| 485 | def _in_memory_dataset(source, schema=None, **kwargs): |
no test coverage detected