(self, path_or_paths, filesystem=None, schema=None, *, filters=None,
read_dictionary=None, binary_type=None, list_type=None,
memory_map=False, buffer_size=None, partitioning="hive",
ignore_prefixes=None,
pre_buffer=True, coerce_int96_timestamp_unit=None,
decryption_properties=None, thrift_string_size_limit=None,
thrift_container_size_limit=None,
page_checksum_verification=False,
arrow_extensions_enabled=True)
| 1385 | """ |
| 1386 | |
| 1387 | def __init__(self, path_or_paths, filesystem=None, schema=None, *, filters=None, |
| 1388 | read_dictionary=None, binary_type=None, list_type=None, |
| 1389 | memory_map=False, buffer_size=None, partitioning="hive", |
| 1390 | ignore_prefixes=None, |
| 1391 | pre_buffer=True, coerce_int96_timestamp_unit=None, |
| 1392 | decryption_properties=None, thrift_string_size_limit=None, |
| 1393 | thrift_container_size_limit=None, |
| 1394 | page_checksum_verification=False, |
| 1395 | arrow_extensions_enabled=True): |
| 1396 | import pyarrow.dataset as ds |
| 1397 | |
| 1398 | # map format arguments |
| 1399 | read_options = { |
| 1400 | "pre_buffer": pre_buffer, |
| 1401 | "coerce_int96_timestamp_unit": coerce_int96_timestamp_unit, |
| 1402 | "thrift_string_size_limit": thrift_string_size_limit, |
| 1403 | "thrift_container_size_limit": thrift_container_size_limit, |
| 1404 | "page_checksum_verification": page_checksum_verification, |
| 1405 | "arrow_extensions_enabled": arrow_extensions_enabled, |
| 1406 | "binary_type": binary_type, |
| 1407 | "list_type": list_type, |
| 1408 | } |
| 1409 | if buffer_size: |
| 1410 | read_options.update(use_buffered_stream=True, |
| 1411 | buffer_size=buffer_size) |
| 1412 | if read_dictionary is not None: |
| 1413 | read_options.update(dictionary_columns=read_dictionary) |
| 1414 | |
| 1415 | if decryption_properties is not None: |
| 1416 | read_options.update(decryption_properties=decryption_properties) |
| 1417 | |
| 1418 | self._filter_expression = None |
| 1419 | if filters is not None: |
| 1420 | self._filter_expression = filters_to_expression(filters) |
| 1421 | |
| 1422 | # map old filesystems to new one |
| 1423 | if filesystem is not None: |
| 1424 | filesystem = _ensure_filesystem( |
| 1425 | filesystem, use_mmap=memory_map) |
| 1426 | elif filesystem is None and memory_map: |
| 1427 | # if memory_map is specified, assume local file system (string |
| 1428 | # path can in principle be URI for any filesystem) |
| 1429 | filesystem = LocalFileSystem(use_mmap=memory_map) |
| 1430 | |
| 1431 | # This needs to be checked after _ensure_filesystem, because that |
| 1432 | # handles the case of an fsspec LocalFileSystem |
| 1433 | if ( |
| 1434 | hasattr(path_or_paths, "__fspath__") and |
| 1435 | filesystem is not None and |
| 1436 | not isinstance(filesystem, LocalFileSystem) |
| 1437 | ): |
| 1438 | raise TypeError( |
| 1439 | "Path-like objects with __fspath__ must only be used with " |
| 1440 | f"local file systems, not {type(filesystem)}" |
| 1441 | ) |
| 1442 | |
| 1443 | # check for single fragment dataset or dataset directory |
| 1444 | single_file = None |
nothing calls this directly
no test coverage detected