MCPcopy Create free account
hub / github.com/apache/arrow / test_write_table_multiple_fragments

Function test_write_table_multiple_fragments

python/pyarrow/tests/test_dataset.py:4809–4842  ·  view source on GitHub ↗
(tempdir)

Source from the content-addressed store, hash-verified

4807
4808
4809def test_write_table_multiple_fragments(tempdir):
4810 table = pa.table([
4811 pa.array(range(10)), pa.array(random.random() for _ in range(10)),
4812 pa.array(['a'] * 5 + ['b'] * 5)
4813 ], names=["f1", "f2", "part"])
4814 table = pa.concat_tables([table]*2)
4815
4816 # Table with multiple batches written as single Fragment by default
4817 base_dir = tempdir / 'single'
4818 ds.write_dataset(table, base_dir, format="feather")
4819 assert set(base_dir.rglob("*")) == set([base_dir / "part-0.feather"])
4820 assert ds.dataset(base_dir, format="ipc").to_table().equals(table)
4821
4822 # Same for single-element list of Table
4823 base_dir = tempdir / 'single-list'
4824 ds.write_dataset([table], base_dir, format="feather")
4825 assert set(base_dir.rglob("*")) == set([base_dir / "part-0.feather"])
4826 assert ds.dataset(base_dir, format="ipc").to_table().equals(table)
4827
4828 # Provide list of batches to write multiple fragments
4829 base_dir = tempdir / 'multiple'
4830 ds.write_dataset(table.to_batches(), base_dir, format="feather")
4831 assert set(base_dir.rglob("*")) == set(
4832 [base_dir / "part-0.feather"])
4833 assert ds.dataset(base_dir, format="ipc").to_table().equals(table)
4834
4835 # Provide list of tables to write multiple fragments
4836 base_dir = tempdir / 'multiple-table'
4837 ds.write_dataset([table, table], base_dir, format="feather")
4838 assert set(base_dir.rglob("*")) == set(
4839 [base_dir / "part-0.feather"])
4840 assert ds.dataset(base_dir, format="ipc").to_table().equals(
4841 pa.concat_tables([table]*2)
4842 )
4843
4844
4845def test_write_iterable(tempdir):

Callers

nothing calls this directly

Calls 4

equalsMethod · 0.80
to_batchesMethod · 0.80
arrayMethod · 0.45
to_tableMethod · 0.45

Tested by

no test coverage detected