(tempdir)
| 4706 | @pytest.mark.parquet |
| 4707 | @pytest.mark.pandas |
| 4708 | def test_write_dataset_use_threads(tempdir): |
| 4709 | directory = tempdir / "partitioned" |
| 4710 | _ = _create_parquet_dataset_partitioned(directory) |
| 4711 | dataset = ds.dataset(directory, partitioning="hive") |
| 4712 | |
| 4713 | partitioning = ds.partitioning( |
| 4714 | pa.schema([("part", pa.string())]), flavor="hive") |
| 4715 | |
| 4716 | target1 = tempdir / 'partitioned1' |
| 4717 | paths_written = [] |
| 4718 | |
| 4719 | def file_visitor(written_file): |
| 4720 | paths_written.append(written_file.path) |
| 4721 | |
| 4722 | ds.write_dataset( |
| 4723 | dataset, target1, format="feather", partitioning=partitioning, |
| 4724 | use_threads=True, file_visitor=file_visitor |
| 4725 | ) |
| 4726 | |
| 4727 | expected_paths = { |
| 4728 | target1 / 'part=a' / 'part-0.feather', |
| 4729 | target1 / 'part=b' / 'part-0.feather' |
| 4730 | } |
| 4731 | paths_written_set = set(map(pathlib.Path, paths_written)) |
| 4732 | assert paths_written_set == expected_paths |
| 4733 | |
| 4734 | target2 = tempdir / 'partitioned2' |
| 4735 | ds.write_dataset( |
| 4736 | dataset, target2, format="feather", partitioning=partitioning, |
| 4737 | use_threads=False |
| 4738 | ) |
| 4739 | |
| 4740 | # check that reading in gives same result |
| 4741 | result1 = ds.dataset(target1, format="feather", partitioning=partitioning) |
| 4742 | result2 = ds.dataset(target2, format="feather", partitioning=partitioning) |
| 4743 | assert result1.to_table().equals(result2.to_table()) |
| 4744 | |
| 4745 | |
| 4746 | @pytest.mark.parquet |
nothing calls this directly
no test coverage detected