MCPcopy Create free account
hub / github.com/apache/arrow / test_write_dataset_max_open_files

Function test_write_dataset_max_open_files

python/pyarrow/tests/test_dataset.py:4612–4677  ·  view source on GitHub ↗
(tempdir)

Source from the content-addressed store, hash-verified

4610
4611@pytest.mark.parquet
4612def test_write_dataset_max_open_files(tempdir):
4613 directory = tempdir / 'ds'
4614 file_format = "parquet"
4615 partition_column_id = 1
4616 column_names = ['c1', 'c2']
4617 record_batch_1 = pa.record_batch(data=[[1, 2, 3, 4, 0, 10],
4618 ['a', 'b', 'c', 'd', 'e', 'a']],
4619 names=column_names)
4620 record_batch_2 = pa.record_batch(data=[[5, 6, 7, 8, 0, 1],
4621 ['a', 'b', 'c', 'd', 'e', 'c']],
4622 names=column_names)
4623 record_batch_3 = pa.record_batch(data=[[9, 10, 11, 12, 0, 1],
4624 ['a', 'b', 'c', 'd', 'e', 'd']],
4625 names=column_names)
4626 record_batch_4 = pa.record_batch(data=[[13, 14, 15, 16, 0, 1],
4627 ['a', 'b', 'c', 'd', 'e', 'b']],
4628 names=column_names)
4629
4630 table = pa.Table.from_batches([record_batch_1, record_batch_2,
4631 record_batch_3, record_batch_4])
4632
4633 partitioning = ds.partitioning(
4634 pa.schema([(column_names[partition_column_id], pa.string())]),
4635 flavor="hive")
4636
4637 data_source_1 = directory / "default"
4638
4639 ds.write_dataset(data=table, base_dir=data_source_1,
4640 partitioning=partitioning, format=file_format)
4641
4642 # Here we consider the number of unique partitions created when
4643 # partitioning column contains duplicate records.
4644 # Returns: (number_of_files_generated, number_of_partitions)
4645 def _get_compare_pair(data_source, record_batch, file_format, col_id):
4646 num_of_files_generated = _get_num_of_files_generated(
4647 base_directory=data_source, file_format=file_format)
4648 number_of_partitions = len(pa.compute.unique(record_batch[col_id]))
4649 return num_of_files_generated, number_of_partitions
4650
4651 # CASE 1: when max_open_files=default & max_open_files >= num_of_partitions
4652 # In case of a writing to disk via partitioning based on a
4653 # particular column (considering row labels in that column),
4654 # the number of unique rows must be equal
4655 # to the number of files generated
4656
4657 num_of_files_generated, number_of_partitions \
4658 = _get_compare_pair(data_source_1, record_batch_1, file_format,
4659 partition_column_id)
4660 assert num_of_files_generated == number_of_partitions
4661
4662 # CASE 2: when max_open_files > 0 & max_open_files < num_of_partitions
4663 # the number of files generated must be greater than the number of
4664 # partitions
4665
4666 data_source_2 = directory / "max_1"
4667
4668 max_open_files = 3
4669

Callers

nothing calls this directly

Calls 4

_get_compare_pairFunction · 0.85
partitioningMethod · 0.80
schemaMethod · 0.45
stringMethod · 0.45

Tested by

no test coverage detected