(s3_server)
| 5195 | @pytest.mark.parquet |
| 5196 | @pytest.mark.s3 |
| 5197 | def test_write_dataset_s3_put_only(s3_server): |
| 5198 | # [ARROW-15892] Testing the create_dir flag which will restrict |
| 5199 | # creating a new directory for writing a dataset. This is |
| 5200 | # required while writing a dataset in s3 where we have very |
| 5201 | # limited permissions and thus we can directly write the dataset |
| 5202 | # without creating a directory. |
| 5203 | from pyarrow.fs import S3FileSystem |
| 5204 | |
| 5205 | # write dataset with s3 filesystem |
| 5206 | host, port, _, _ = s3_server['connection'] |
| 5207 | |
| 5208 | _configure_s3_limited_user(s3_server, _minio_put_only_policy, |
| 5209 | 'test_dataset_limited_user', 'limited123') |
| 5210 | fs = S3FileSystem( |
| 5211 | access_key='test_dataset_limited_user', |
| 5212 | secret_key='limited123', |
| 5213 | endpoint_override=f'{host}:{port}', |
| 5214 | scheme='http' |
| 5215 | ) |
| 5216 | |
| 5217 | table = pa.table([ |
| 5218 | pa.array(range(20)), pa.array(random.random() for _ in range(20)), |
| 5219 | pa.array(['a']*10 + ['b'] * 10)], |
| 5220 | names=["f1", "f2", "part"] |
| 5221 | ) |
| 5222 | part = ds.partitioning(pa.schema([("part", pa.string())]), flavor="hive") |
| 5223 | |
| 5224 | # writing with filesystem object with create_dir flag set to false |
| 5225 | ds.write_dataset( |
| 5226 | table, "existing-bucket", filesystem=fs, |
| 5227 | format="feather", create_dir=False, partitioning=part, |
| 5228 | existing_data_behavior='overwrite_or_ignore' |
| 5229 | ) |
| 5230 | # check roundtrip |
| 5231 | result = ds.dataset( |
| 5232 | "existing-bucket", filesystem=fs, format="ipc", partitioning="hive" |
| 5233 | ).to_table() |
| 5234 | assert result.equals(table) |
| 5235 | |
| 5236 | # Passing create_dir is fine if the bucket already exists |
| 5237 | ds.write_dataset( |
| 5238 | table, "existing-bucket", filesystem=fs, |
| 5239 | format="feather", create_dir=True, partitioning=part, |
| 5240 | existing_data_behavior='overwrite_or_ignore' |
| 5241 | ) |
| 5242 | # check roundtrip |
| 5243 | result = ds.dataset( |
| 5244 | "existing-bucket", filesystem=fs, format="ipc", partitioning="hive" |
| 5245 | ).to_table() |
| 5246 | assert result.equals(table) |
| 5247 | |
| 5248 | # Error enforced by filesystem |
| 5249 | with pytest.raises(OSError, |
| 5250 | match="Bucket 'non-existing-bucket' not found"): |
| 5251 | ds.write_dataset( |
| 5252 | table, "non-existing-bucket", filesystem=fs, |
| 5253 | format="feather", create_dir=True, |
| 5254 | existing_data_behavior='overwrite_or_ignore' |
nothing calls this directly
no test coverage detected