MCPcopy Create free account
hub / github.com/apache/arrow / test_write_dataset_s3_put_only

Function test_write_dataset_s3_put_only

python/pyarrow/tests/test_dataset.py:5197–5270  ·  view source on GitHub ↗
(s3_server)

Source from the content-addressed store, hash-verified

5195@pytest.mark.parquet
5196@pytest.mark.s3
5197def test_write_dataset_s3_put_only(s3_server):
5198 # [ARROW-15892] Testing the create_dir flag which will restrict
5199 # creating a new directory for writing a dataset. This is
5200 # required while writing a dataset in s3 where we have very
5201 # limited permissions and thus we can directly write the dataset
5202 # without creating a directory.
5203 from pyarrow.fs import S3FileSystem
5204
5205 # write dataset with s3 filesystem
5206 host, port, _, _ = s3_server['connection']
5207
5208 _configure_s3_limited_user(s3_server, _minio_put_only_policy,
5209 'test_dataset_limited_user', 'limited123')
5210 fs = S3FileSystem(
5211 access_key='test_dataset_limited_user',
5212 secret_key='limited123',
5213 endpoint_override=f'{host}:{port}',
5214 scheme='http'
5215 )
5216
5217 table = pa.table([
5218 pa.array(range(20)), pa.array(random.random() for _ in range(20)),
5219 pa.array(['a']*10 + ['b'] * 10)],
5220 names=["f1", "f2", "part"]
5221 )
5222 part = ds.partitioning(pa.schema([("part", pa.string())]), flavor="hive")
5223
5224 # writing with filesystem object with create_dir flag set to false
5225 ds.write_dataset(
5226 table, "existing-bucket", filesystem=fs,
5227 format="feather", create_dir=False, partitioning=part,
5228 existing_data_behavior='overwrite_or_ignore'
5229 )
5230 # check roundtrip
5231 result = ds.dataset(
5232 "existing-bucket", filesystem=fs, format="ipc", partitioning="hive"
5233 ).to_table()
5234 assert result.equals(table)
5235
5236 # Passing create_dir is fine if the bucket already exists
5237 ds.write_dataset(
5238 table, "existing-bucket", filesystem=fs,
5239 format="feather", create_dir=True, partitioning=part,
5240 existing_data_behavior='overwrite_or_ignore'
5241 )
5242 # check roundtrip
5243 result = ds.dataset(
5244 "existing-bucket", filesystem=fs, format="ipc", partitioning="hive"
5245 ).to_table()
5246 assert result.equals(table)
5247
5248 # Error enforced by filesystem
5249 with pytest.raises(OSError,
5250 match="Bucket 'non-existing-bucket' not found"):
5251 ds.write_dataset(
5252 table, "non-existing-bucket", filesystem=fs,
5253 format="feather", create_dir=True,
5254 existing_data_behavior='overwrite_or_ignore'

Callers

nothing calls this directly

Calls 8

S3FileSystemClass · 0.85
partitioningMethod · 0.80
equalsMethod · 0.80
arrayMethod · 0.45
schemaMethod · 0.45
stringMethod · 0.45
to_tableMethod · 0.45

Tested by

no test coverage detected