MCPcopy Create free account
hub / github.com/apache/arrow / test_parquet_scan_options

Function test_parquet_scan_options

python/pyarrow/tests/test_dataset.py:1011–1070  ·  view source on GitHub ↗
()

Source from the content-addressed store, hash-verified

1009
1010@pytest.mark.parquet
1011def test_parquet_scan_options():
1012 opts1 = ds.ParquetFragmentScanOptions()
1013 opts2 = ds.ParquetFragmentScanOptions(buffer_size=4096)
1014 opts3 = ds.ParquetFragmentScanOptions(
1015 buffer_size=2**13, use_buffered_stream=True)
1016 opts4 = ds.ParquetFragmentScanOptions(buffer_size=2**13, pre_buffer=False)
1017 opts5 = ds.ParquetFragmentScanOptions(
1018 thrift_string_size_limit=123456,
1019 thrift_container_size_limit=987654,)
1020 opts6 = ds.ParquetFragmentScanOptions(
1021 page_checksum_verification=True)
1022 cache_opts = pa.CacheOptions(
1023 hole_size_limit=2**10, range_size_limit=8*2**10, lazy=True)
1024 opts7 = ds.ParquetFragmentScanOptions(pre_buffer=True, cache_options=cache_opts)
1025 opts8 = ds.ParquetFragmentScanOptions(schema_depth_limit=42)
1026
1027 assert opts1.use_buffered_stream is False
1028 assert opts1.buffer_size == 2**13
1029 if is_threading_enabled(): # pre buffering requires threads
1030 assert opts1.pre_buffer is True
1031 assert opts1.thrift_string_size_limit == 100_000_000 # default in C++
1032 assert opts1.thrift_container_size_limit == 1_000_000 # default in C++
1033 assert opts1.page_checksum_verification is False
1034 assert opts1.schema_depth_limit == 100 # default in C++
1035
1036 assert opts2.use_buffered_stream is False
1037 assert opts2.buffer_size == 2**12
1038 if is_threading_enabled(): # pre buffering requires threads
1039 assert opts2.pre_buffer is True
1040
1041 assert opts3.use_buffered_stream is True
1042 assert opts3.buffer_size == 2**13
1043 if is_threading_enabled(): # pre buffering requires threads
1044 assert opts3.pre_buffer is True
1045
1046 assert opts4.use_buffered_stream is False
1047 assert opts4.buffer_size == 2**13
1048 if is_threading_enabled(): # pre buffering requires threads
1049 assert opts4.pre_buffer is False
1050
1051 assert opts5.thrift_string_size_limit == 123456
1052 assert opts5.thrift_container_size_limit == 987654
1053
1054 assert opts6.page_checksum_verification is True
1055
1056 if is_threading_enabled(): # pre buffering requires threads
1057 assert opts7.pre_buffer is True
1058 assert opts7.cache_options == cache_opts
1059 assert opts7.cache_options != opts1.cache_options
1060
1061 assert opts8.schema_depth_limit == 42
1062
1063 assert opts1 == opts1
1064 assert opts1 != opts2
1065 assert opts2 != opts3
1066 assert opts3 != opts4
1067 assert opts5 != opts1
1068 assert opts6 != opts1

Callers

nothing calls this directly

Calls 1

Tested by

no test coverage detected