| 5285 | |
| 5286 | @pytest.mark.large_memory |
| 5287 | def test_nested_chunking_valid(): |
| 5288 | # GH-32439: Chunking can cause arrays to be in invalid state |
| 5289 | # when nested types are involved. |
| 5290 | # Here we simply ensure we validate correctly. |
| 5291 | |
| 5292 | def roundtrip(df, schema=None): |
| 5293 | tab = pa.Table.from_pandas(df, schema=schema) |
| 5294 | tab.validate(full=True) |
| 5295 | # we expect to trigger chunking internally |
| 5296 | # an assertion failure here may just mean this threshold has changed |
| 5297 | num_chunks = tab.column(0).num_chunks |
| 5298 | assert num_chunks > 1 |
| 5299 | tm.assert_frame_equal(tab.to_pandas(self_destruct=True, |
| 5300 | maps_as_pydicts="strict"), df) |
| 5301 | |
| 5302 | x = b"0" * 720000000 |
| 5303 | roundtrip(pd.DataFrame({"strings": [x, x, x]})) |
| 5304 | |
| 5305 | struct = {"struct_field": x} |
| 5306 | roundtrip(pd.DataFrame({"structs": [struct, struct, struct]})) |
| 5307 | |
| 5308 | lists = [x] |
| 5309 | roundtrip(pd.DataFrame({"lists": [lists, lists, lists]})) |
| 5310 | |
| 5311 | los = [struct] |
| 5312 | roundtrip(pd.DataFrame({"los": [los, los, los]})) |
| 5313 | |
| 5314 | sol = {"struct_field": lists} |
| 5315 | roundtrip(pd.DataFrame({"sol": [sol, sol, sol]})) |
| 5316 | |
| 5317 | map_of_los = {"a": los} |
| 5318 | map_type = pa.map_(pa.string(), |
| 5319 | pa.list_(pa.struct([("struct_field", pa.binary())]))) |
| 5320 | schema = pa.schema([("maps", map_type)]) |
| 5321 | roundtrip(pd.DataFrame({"maps": [map_of_los, map_of_los, map_of_los]}), |
| 5322 | schema=schema) |
| 5323 | |
| 5324 | |
| 5325 | def test_bytes_column_name_to_pandas(): |