(tempdir)
| 4700 | @pytest.mark.parquet |
| 4701 | @pytest.mark.pandas |
| 4702 | def test_write_dataset_use_threads(tempdir): |
| 4703 | directory = tempdir / "partitioned" |
| 4704 | _ = _create_parquet_dataset_partitioned(directory) |
| 4705 | dataset = ds.dataset(directory, partitioning="hive") |
| 4706 | |
| 4707 | partitioning = ds.partitioning( |
| 4708 | pa.schema([("part", pa.string())]), flavor="hive") |
| 4709 | |
| 4710 | target1 = tempdir / 'partitioned1' |
| 4711 | paths_written = [] |
| 4712 | |
| 4713 | def file_visitor(written_file): |
| 4714 | paths_written.append(written_file.path) |
| 4715 | |
| 4716 | ds.write_dataset( |
| 4717 | dataset, target1, format="feather", partitioning=partitioning, |
| 4718 | use_threads=True, file_visitor=file_visitor |
| 4719 | ) |
| 4720 | |
| 4721 | expected_paths = { |
| 4722 | target1 / 'part=a' / 'part-0.feather', |
| 4723 | target1 / 'part=b' / 'part-0.feather' |
| 4724 | } |
| 4725 | paths_written_set = set(map(pathlib.Path, paths_written)) |
| 4726 | assert paths_written_set == expected_paths |
| 4727 | |
| 4728 | target2 = tempdir / 'partitioned2' |
| 4729 | ds.write_dataset( |
| 4730 | dataset, target2, format="feather", partitioning=partitioning, |
| 4731 | use_threads=False |
| 4732 | ) |
| 4733 | |
| 4734 | # check that reading in gives same result |
| 4735 | result1 = ds.dataset(target1, format="feather", partitioning=partitioning) |
| 4736 | result2 = ds.dataset(target2, format="feather", partitioning=partitioning) |
| 4737 | assert result1.to_table().equals(result2.to_table()) |
| 4738 | |
| 4739 | |
| 4740 | @pytest.mark.parquet |
nothing calls this directly
no test coverage detected