MCPcopy Create free account
hub / github.com/apache/arrow / test_write_table

Function test_write_table

python/pyarrow/tests/test_dataset.py:4762–4806  ·  view source on GitHub ↗
(tempdir)

Source from the content-addressed store, hash-verified

4760
4761
4762def test_write_table(tempdir):
4763 table = pa.table([
4764 pa.array(range(20)), pa.array(random.random() for _ in range(20)),
4765 pa.array(['a'] * 10 + ['b'] * 10)
4766 ], names=["f1", "f2", "part"])
4767
4768 base_dir = tempdir / 'single'
4769 ds.write_dataset(table, base_dir,
4770 basename_template='dat_{i}.arrow', format="feather")
4771 # check that all files are present
4772 file_paths = list(base_dir.rglob("*"))
4773 expected_paths = [base_dir / "dat_0.arrow"]
4774 assert set(file_paths) == set(expected_paths)
4775 # check Table roundtrip
4776 result = ds.dataset(base_dir, format="ipc").to_table()
4777 assert result.equals(table)
4778
4779 # with partitioning
4780 base_dir = tempdir / 'partitioned'
4781 expected_paths = [
4782 base_dir / "part=a", base_dir / "part=a" / "dat_0.arrow",
4783 base_dir / "part=b", base_dir / "part=b" / "dat_0.arrow"
4784 ]
4785
4786 visited_paths = []
4787 visited_sizes = []
4788
4789 def file_visitor(written_file):
4790 visited_paths.append(written_file.path)
4791 visited_sizes.append(written_file.size)
4792
4793 partitioning = ds.partitioning(
4794 pa.schema([("part", pa.string())]), flavor="hive")
4795 ds.write_dataset(table, base_dir, format="feather",
4796 basename_template='dat_{i}.arrow',
4797 partitioning=partitioning, file_visitor=file_visitor)
4798 file_paths = list(base_dir.rglob("*"))
4799 assert set(file_paths) == set(expected_paths)
4800 actual_sizes = [os.path.getsize(path) for path in visited_paths]
4801 assert visited_sizes == actual_sizes
4802 result = ds.dataset(base_dir, format="ipc", partitioning=partitioning)
4803 assert result.to_table().equals(table)
4804 assert len(visited_paths) == 2
4805 for visited_path in visited_paths:
4806 assert pathlib.Path(visited_path) in expected_paths
4807
4808
4809def test_write_table_multiple_fragments(tempdir):

Callers

nothing calls this directly

Calls 9

listFunction · 0.85
lenFunction · 0.85
equalsMethod · 0.80
partitioningMethod · 0.80
arrayMethod · 0.45
to_tableMethod · 0.45
schemaMethod · 0.45
stringMethod · 0.45
PathMethod · 0.45

Tested by

no test coverage detected