| 4754 | |
| 4755 | |
| 4756 | def test_write_table(tempdir): |
| 4757 | table = pa.table([ |
| 4758 | pa.array(range(20)), pa.array(random.random() for _ in range(20)), |
| 4759 | pa.array(['a'] * 10 + ['b'] * 10) |
| 4760 | ], names=["f1", "f2", "part"]) |
| 4761 | |
| 4762 | base_dir = tempdir / 'single' |
| 4763 | ds.write_dataset(table, base_dir, |
| 4764 | basename_template='dat_{i}.arrow', format="feather") |
| 4765 | # check that all files are present |
| 4766 | file_paths = list(base_dir.rglob("*")) |
| 4767 | expected_paths = [base_dir / "dat_0.arrow"] |
| 4768 | assert set(file_paths) == set(expected_paths) |
| 4769 | # check Table roundtrip |
| 4770 | result = ds.dataset(base_dir, format="ipc").to_table() |
| 4771 | assert result.equals(table) |
| 4772 | |
| 4773 | # with partitioning |
| 4774 | base_dir = tempdir / 'partitioned' |
| 4775 | expected_paths = [ |
| 4776 | base_dir / "part=a", base_dir / "part=a" / "dat_0.arrow", |
| 4777 | base_dir / "part=b", base_dir / "part=b" / "dat_0.arrow" |
| 4778 | ] |
| 4779 | |
| 4780 | visited_paths = [] |
| 4781 | visited_sizes = [] |
| 4782 | |
| 4783 | def file_visitor(written_file): |
| 4784 | visited_paths.append(written_file.path) |
| 4785 | visited_sizes.append(written_file.size) |
| 4786 | |
| 4787 | partitioning = ds.partitioning( |
| 4788 | pa.schema([("part", pa.string())]), flavor="hive") |
| 4789 | ds.write_dataset(table, base_dir, format="feather", |
| 4790 | basename_template='dat_{i}.arrow', |
| 4791 | partitioning=partitioning, file_visitor=file_visitor) |
| 4792 | file_paths = list(base_dir.rglob("*")) |
| 4793 | assert set(file_paths) == set(expected_paths) |
| 4794 | actual_sizes = [os.path.getsize(path) for path in visited_paths] |
| 4795 | assert visited_sizes == actual_sizes |
| 4796 | result = ds.dataset(base_dir, format="ipc", partitioning=partitioning) |
| 4797 | assert result.to_table().equals(table) |
| 4798 | assert len(visited_paths) == 2 |
| 4799 | for visited_path in visited_paths: |
| 4800 | assert pathlib.Path(visited_path) in expected_paths |
| 4801 | |
| 4802 | |
| 4803 | def test_write_table_multiple_fragments(tempdir): |