(tempdir)
| 4703 | @pytest.mark.parquet |
| 4704 | @pytest.mark.pandas |
| 4705 | def test_write_dataset_use_threads(tempdir): |
| 4706 | directory = tempdir / "partitioned" |
| 4707 | _ = _create_parquet_dataset_partitioned(directory) |
| 4708 | dataset = ds.dataset(directory, partitioning="hive") |
| 4709 | |
| 4710 | partitioning = ds.partitioning( |
| 4711 | pa.schema([("part", pa.string())]), flavor="hive") |
| 4712 | |
| 4713 | target1 = tempdir / 'partitioned1' |
| 4714 | paths_written = [] |
| 4715 | |
| 4716 | def file_visitor(written_file): |
| 4717 | paths_written.append(written_file.path) |
| 4718 | |
| 4719 | ds.write_dataset( |
| 4720 | dataset, target1, format="feather", partitioning=partitioning, |
| 4721 | use_threads=True, file_visitor=file_visitor |
| 4722 | ) |
| 4723 | |
| 4724 | expected_paths = { |
| 4725 | target1 / 'part=a' / 'part-0.feather', |
| 4726 | target1 / 'part=b' / 'part-0.feather' |
| 4727 | } |
| 4728 | paths_written_set = set(map(pathlib.Path, paths_written)) |
| 4729 | assert paths_written_set == expected_paths |
| 4730 | |
| 4731 | target2 = tempdir / 'partitioned2' |
| 4732 | ds.write_dataset( |
| 4733 | dataset, target2, format="feather", partitioning=partitioning, |
| 4734 | use_threads=False |
| 4735 | ) |
| 4736 | |
| 4737 | # check that reading in gives same result |
| 4738 | result1 = ds.dataset(target1, format="feather", partitioning=partitioning) |
| 4739 | result2 = ds.dataset(target2, format="feather", partitioning=partitioning) |
| 4740 | assert result1.to_table().equals(result2.to_table()) |
| 4741 | |
| 4742 | |
| 4743 | @pytest.mark.parquet |
nothing calls this directly
no test coverage detected