(self, path_or_paths, filesystem=None, schema=None, *, filters=None,
read_dictionary=None, binary_type=None, list_type=None,
memory_map=False, buffer_size=None, partitioning="hive",
ignore_prefixes=None,
pre_buffer=True, coerce_int96_timestamp_unit=None,
decryption_properties=None, thrift_string_size_limit=None,
thrift_container_size_limit=None, schema_depth_limit=None,
page_checksum_verification=False,
arrow_extensions_enabled=True)
| 1399 | """ |
| 1400 | |
| 1401 | def __init__(self, path_or_paths, filesystem=None, schema=None, *, filters=None, |
| 1402 | read_dictionary=None, binary_type=None, list_type=None, |
| 1403 | memory_map=False, buffer_size=None, partitioning="hive", |
| 1404 | ignore_prefixes=None, |
| 1405 | pre_buffer=True, coerce_int96_timestamp_unit=None, |
| 1406 | decryption_properties=None, thrift_string_size_limit=None, |
| 1407 | thrift_container_size_limit=None, schema_depth_limit=None, |
| 1408 | page_checksum_verification=False, |
| 1409 | arrow_extensions_enabled=True): |
| 1410 | import pyarrow.dataset as ds |
| 1411 | |
| 1412 | # map format arguments |
| 1413 | read_options = { |
| 1414 | "pre_buffer": pre_buffer, |
| 1415 | "coerce_int96_timestamp_unit": coerce_int96_timestamp_unit, |
| 1416 | "thrift_string_size_limit": thrift_string_size_limit, |
| 1417 | "thrift_container_size_limit": thrift_container_size_limit, |
| 1418 | "schema_depth_limit": schema_depth_limit, |
| 1419 | "page_checksum_verification": page_checksum_verification, |
| 1420 | "arrow_extensions_enabled": arrow_extensions_enabled, |
| 1421 | "binary_type": binary_type, |
| 1422 | "list_type": list_type, |
| 1423 | } |
| 1424 | if buffer_size: |
| 1425 | read_options.update(use_buffered_stream=True, |
| 1426 | buffer_size=buffer_size) |
| 1427 | if read_dictionary is not None: |
| 1428 | read_options.update(dictionary_columns=read_dictionary) |
| 1429 | |
| 1430 | if decryption_properties is not None: |
| 1431 | read_options.update(decryption_properties=decryption_properties) |
| 1432 | |
| 1433 | self._filter_expression = None |
| 1434 | if filters is not None: |
| 1435 | self._filter_expression = filters_to_expression(filters) |
| 1436 | |
| 1437 | # map old filesystems to new one |
| 1438 | if filesystem is not None: |
| 1439 | filesystem = _ensure_filesystem( |
| 1440 | filesystem, use_mmap=memory_map) |
| 1441 | elif filesystem is None and memory_map: |
| 1442 | # if memory_map is specified, assume local file system (string |
| 1443 | # path can in principle be URI for any filesystem) |
| 1444 | filesystem = LocalFileSystem(use_mmap=memory_map) |
| 1445 | |
| 1446 | # This needs to be checked after _ensure_filesystem, because that |
| 1447 | # handles the case of an fsspec LocalFileSystem |
| 1448 | if ( |
| 1449 | hasattr(path_or_paths, "__fspath__") and |
| 1450 | filesystem is not None and |
| 1451 | not isinstance(filesystem, LocalFileSystem) |
| 1452 | ): |
| 1453 | raise TypeError( |
| 1454 | "Path-like objects with __fspath__ must only be used with " |
| 1455 | f"local file systems, not {type(filesystem)}" |
| 1456 | ) |
| 1457 | |
| 1458 | # check for single fragment dataset or dataset directory |
nothing calls this directly
no test coverage detected