MCPcopy Create free account
hub / github.com/apache/arrow / test_write_dataset_s3

Function test_write_dataset_s3

python/pyarrow/tests/test_dataset.py:5130–5174  ·  view source on GitHub ↗
(s3_example_simple)

Source from the content-addressed store, hash-verified

5128@pytest.mark.parquet
5129@pytest.mark.s3
5130def test_write_dataset_s3(s3_example_simple):
5131 # write dataset with s3 filesystem
5132 _, _, fs, _, host, port, access_key, secret_key = s3_example_simple
5133 uri_template = (
5134 "s3://{}:{}@{{}}?scheme=http&endpoint_override={}:{}".format(
5135 access_key, secret_key, host, port)
5136 )
5137
5138 table = pa.table([
5139 pa.array(range(20)), pa.array(random.random() for _ in range(20)),
5140 pa.array(['a'] * 10 + ['b'] * 10)],
5141 names=["f1", "f2", "part"]
5142 )
5143 part = ds.partitioning(pa.schema([("part", pa.string())]), flavor="hive")
5144
5145 # writing with filesystem object
5146 ds.write_dataset(
5147 table, "mybucket/dataset", filesystem=fs, format="feather",
5148 partitioning=part
5149 )
5150 # check roundtrip
5151 result = ds.dataset(
5152 "mybucket/dataset", filesystem=fs, format="ipc", partitioning="hive"
5153 ).to_table()
5154 assert result.equals(table)
5155
5156 # writing with URI
5157 uri = uri_template.format("mybucket/dataset2")
5158 ds.write_dataset(table, uri, format="feather", partitioning=part)
5159 # check roundtrip
5160 result = ds.dataset(
5161 "mybucket/dataset2", filesystem=fs, format="ipc", partitioning="hive"
5162 ).to_table()
5163 assert result.equals(table)
5164
5165 # writing with path + URI as filesystem
5166 uri = uri_template.format("mybucket")
5167 ds.write_dataset(
5168 table, "dataset3", filesystem=uri, format="feather", partitioning=part
5169 )
5170 # check roundtrip
5171 result = ds.dataset(
5172 "mybucket/dataset3", filesystem=fs, format="ipc", partitioning="hive"
5173 ).to_table()
5174 assert result.equals(table)
5175
5176
5177_minio_put_only_policy = """{

Callers

nothing calls this directly

Calls 7

partitioningMethod · 0.80
equalsMethod · 0.80
formatMethod · 0.45
arrayMethod · 0.45
schemaMethod · 0.45
stringMethod · 0.45
to_tableMethod · 0.45

Tested by

no test coverage detected