(tempdir, compression, dataset_reader)
| 3740 | "brotli" # not supported |
| 3741 | ]) |
| 3742 | def test_feather_format_compressed(tempdir, compression, dataset_reader): |
| 3743 | table = pa.table({'a': pa.array([0]*300, type="int8"), |
| 3744 | 'b': pa.array([.1, .2, .3]*100, type="float64")}) |
| 3745 | if not pa.Codec.is_available(compression): |
| 3746 | pytest.skip() |
| 3747 | |
| 3748 | basedir = tempdir / "feather_dataset_compressed" |
| 3749 | basedir.mkdir() |
| 3750 | file_format = ds.IpcFileFormat() |
| 3751 | |
| 3752 | uncompressed_basedir = tempdir / "feather_dataset_uncompressed" |
| 3753 | uncompressed_basedir.mkdir() |
| 3754 | ds.write_dataset( |
| 3755 | table, |
| 3756 | str(uncompressed_basedir / "data.arrow"), |
| 3757 | format=file_format, |
| 3758 | file_options=file_format.make_write_options(compression=None) |
| 3759 | ) |
| 3760 | |
| 3761 | if compression == "brotli": |
| 3762 | with pytest.raises(ValueError, match="Compression type"): |
| 3763 | write_options = file_format.make_write_options( |
| 3764 | compression=compression) |
| 3765 | with pytest.raises(ValueError, match="Compression type"): |
| 3766 | codec = pa.Codec(compression) |
| 3767 | write_options = file_format.make_write_options(compression=codec) |
| 3768 | return |
| 3769 | |
| 3770 | write_options = file_format.make_write_options(compression=compression) |
| 3771 | ds.write_dataset( |
| 3772 | table, |
| 3773 | str(basedir / "data.arrow"), |
| 3774 | format=file_format, |
| 3775 | file_options=write_options |
| 3776 | ) |
| 3777 | |
| 3778 | dataset = ds.dataset(basedir, format=ds.IpcFileFormat()) |
| 3779 | result = dataset_reader.to_table(dataset) |
| 3780 | assert result.equals(table) |
| 3781 | |
| 3782 | compressed_file = basedir / "data.arrow" / "part-0.arrow" |
| 3783 | compressed_size = compressed_file.stat().st_size |
| 3784 | uncompressed_file = uncompressed_basedir / "data.arrow" / "part-0.arrow" |
| 3785 | uncompressed_size = uncompressed_file.stat().st_size |
| 3786 | assert compressed_size < uncompressed_size |
| 3787 | |
| 3788 | |
| 3789 | def _create_parquet_dataset_simple(root_path): |
nothing calls this directly
no test coverage detected