| 5128 | @pytest.mark.parquet |
| 5129 | @pytest.mark.s3 |
| 5130 | def test_write_dataset_s3(s3_example_simple): |
| 5131 | # write dataset with s3 filesystem |
| 5132 | _, _, fs, _, host, port, access_key, secret_key = s3_example_simple |
| 5133 | uri_template = ( |
| 5134 | "s3://{}:{}@{{}}?scheme=http&endpoint_override={}:{}".format( |
| 5135 | access_key, secret_key, host, port) |
| 5136 | ) |
| 5137 | |
| 5138 | table = pa.table([ |
| 5139 | pa.array(range(20)), pa.array(random.random() for _ in range(20)), |
| 5140 | pa.array(['a'] * 10 + ['b'] * 10)], |
| 5141 | names=["f1", "f2", "part"] |
| 5142 | ) |
| 5143 | part = ds.partitioning(pa.schema([("part", pa.string())]), flavor="hive") |
| 5144 | |
| 5145 | # writing with filesystem object |
| 5146 | ds.write_dataset( |
| 5147 | table, "mybucket/dataset", filesystem=fs, format="feather", |
| 5148 | partitioning=part |
| 5149 | ) |
| 5150 | # check roundtrip |
| 5151 | result = ds.dataset( |
| 5152 | "mybucket/dataset", filesystem=fs, format="ipc", partitioning="hive" |
| 5153 | ).to_table() |
| 5154 | assert result.equals(table) |
| 5155 | |
| 5156 | # writing with URI |
| 5157 | uri = uri_template.format("mybucket/dataset2") |
| 5158 | ds.write_dataset(table, uri, format="feather", partitioning=part) |
| 5159 | # check roundtrip |
| 5160 | result = ds.dataset( |
| 5161 | "mybucket/dataset2", filesystem=fs, format="ipc", partitioning="hive" |
| 5162 | ).to_table() |
| 5163 | assert result.equals(table) |
| 5164 | |
| 5165 | # writing with path + URI as filesystem |
| 5166 | uri = uri_template.format("mybucket") |
| 5167 | ds.write_dataset( |
| 5168 | table, "dataset3", filesystem=uri, format="feather", partitioning=part |
| 5169 | ) |
| 5170 | # check roundtrip |
| 5171 | result = ds.dataset( |
| 5172 | "mybucket/dataset3", filesystem=fs, format="ipc", partitioning="hive" |
| 5173 | ).to_table() |
| 5174 | assert result.equals(table) |
| 5175 | |
| 5176 | |
| 5177 | _minio_put_only_policy = """{ |