(tempdir, compression, dataset_reader)
| 3745 | "brotli" # not supported |
| 3746 | ]) |
| 3747 | def test_feather_format_compressed(tempdir, compression, dataset_reader): |
| 3748 | table = pa.table({'a': pa.array([0]*300, type="int8"), |
| 3749 | 'b': pa.array([.1, .2, .3]*100, type="float64")}) |
| 3750 | if not pa.Codec.is_available(compression): |
| 3751 | pytest.skip() |
| 3752 | |
| 3753 | basedir = tempdir / "feather_dataset_compressed" |
| 3754 | basedir.mkdir() |
| 3755 | file_format = ds.IpcFileFormat() |
| 3756 | |
| 3757 | uncompressed_basedir = tempdir / "feather_dataset_uncompressed" |
| 3758 | uncompressed_basedir.mkdir() |
| 3759 | ds.write_dataset( |
| 3760 | table, |
| 3761 | str(uncompressed_basedir / "data.arrow"), |
| 3762 | format=file_format, |
| 3763 | file_options=file_format.make_write_options(compression=None) |
| 3764 | ) |
| 3765 | |
| 3766 | if compression == "brotli": |
| 3767 | with pytest.raises(ValueError, match="Compression type"): |
| 3768 | write_options = file_format.make_write_options( |
| 3769 | compression=compression) |
| 3770 | with pytest.raises(ValueError, match="Compression type"): |
| 3771 | codec = pa.Codec(compression) |
| 3772 | write_options = file_format.make_write_options(compression=codec) |
| 3773 | return |
| 3774 | |
| 3775 | write_options = file_format.make_write_options(compression=compression) |
| 3776 | ds.write_dataset( |
| 3777 | table, |
| 3778 | str(basedir / "data.arrow"), |
| 3779 | format=file_format, |
| 3780 | file_options=write_options |
| 3781 | ) |
| 3782 | |
| 3783 | dataset = ds.dataset(basedir, format=ds.IpcFileFormat()) |
| 3784 | result = dataset_reader.to_table(dataset) |
| 3785 | assert result.equals(table) |
| 3786 | |
| 3787 | compressed_file = basedir / "data.arrow" / "part-0.arrow" |
| 3788 | compressed_size = compressed_file.stat().st_size |
| 3789 | uncompressed_file = uncompressed_basedir / "data.arrow" / "part-0.arrow" |
| 3790 | uncompressed_size = uncompressed_file.stat().st_size |
| 3791 | assert compressed_size < uncompressed_size |
| 3792 | |
| 3793 | |
| 3794 | def _create_parquet_dataset_simple(root_path): |
nothing calls this directly
no test coverage detected