| 5350 | |
| 5351 | @pytest.mark.large_memory |
| 5352 | def test_nested_chunking_valid(): |
| 5353 | # GH-32439: Chunking can cause arrays to be in invalid state |
| 5354 | # when nested types are involved. |
| 5355 | # Here we simply ensure we validate correctly. |
| 5356 | |
| 5357 | def roundtrip(df, schema=None): |
| 5358 | tab = pa.Table.from_pandas(df, schema=schema) |
| 5359 | tab.validate(full=True) |
| 5360 | # we expect to trigger chunking internally |
| 5361 | # an assertion failure here may just mean this threshold has changed |
| 5362 | num_chunks = tab.column(0).num_chunks |
| 5363 | assert num_chunks > 1 |
| 5364 | tm.assert_frame_equal(tab.to_pandas(self_destruct=True, |
| 5365 | maps_as_pydicts="strict"), df) |
| 5366 | |
| 5367 | x = b"0" * 720000000 |
| 5368 | roundtrip(pd.DataFrame({"strings": [x, x, x]})) |
| 5369 | |
| 5370 | struct = {"struct_field": x} |
| 5371 | roundtrip(pd.DataFrame({"structs": [struct, struct, struct]})) |
| 5372 | |
| 5373 | lists = [x] |
| 5374 | roundtrip(pd.DataFrame({"lists": [lists, lists, lists]})) |
| 5375 | |
| 5376 | los = [struct] |
| 5377 | roundtrip(pd.DataFrame({"los": [los, los, los]})) |
| 5378 | |
| 5379 | sol = {"struct_field": lists} |
| 5380 | roundtrip(pd.DataFrame({"sol": [sol, sol, sol]})) |
| 5381 | |
| 5382 | map_of_los = {"a": los} |
| 5383 | map_type = pa.map_(pa.string(), |
| 5384 | pa.list_(pa.struct([("struct_field", pa.binary())]))) |
| 5385 | schema = pa.schema([("maps", map_type)]) |
| 5386 | roundtrip(pd.DataFrame({"maps": [map_of_los, map_of_los, map_of_los]}), |
| 5387 | schema=schema) |
| 5388 | |
| 5389 | |
| 5390 | def test_bytes_column_name_to_pandas(): |