| 5273 | |
| 5274 | @pytest.mark.large_memory |
| 5275 | def test_nested_chunking_valid(): |
| 5276 | # GH-32439: Chunking can cause arrays to be in invalid state |
| 5277 | # when nested types are involved. |
| 5278 | # Here we simply ensure we validate correctly. |
| 5279 | |
| 5280 | def roundtrip(df, schema=None): |
| 5281 | tab = pa.Table.from_pandas(df, schema=schema) |
| 5282 | tab.validate(full=True) |
| 5283 | # we expect to trigger chunking internally |
| 5284 | # an assertion failure here may just mean this threshold has changed |
| 5285 | num_chunks = tab.column(0).num_chunks |
| 5286 | assert num_chunks > 1 |
| 5287 | tm.assert_frame_equal(tab.to_pandas(self_destruct=True, |
| 5288 | maps_as_pydicts="strict"), df) |
| 5289 | |
| 5290 | x = b"0" * 720000000 |
| 5291 | roundtrip(pd.DataFrame({"strings": [x, x, x]})) |
| 5292 | |
| 5293 | struct = {"struct_field": x} |
| 5294 | roundtrip(pd.DataFrame({"structs": [struct, struct, struct]})) |
| 5295 | |
| 5296 | lists = [x] |
| 5297 | roundtrip(pd.DataFrame({"lists": [lists, lists, lists]})) |
| 5298 | |
| 5299 | los = [struct] |
| 5300 | roundtrip(pd.DataFrame({"los": [los, los, los]})) |
| 5301 | |
| 5302 | sol = {"struct_field": lists} |
| 5303 | roundtrip(pd.DataFrame({"sol": [sol, sol, sol]})) |
| 5304 | |
| 5305 | map_of_los = {"a": los} |
| 5306 | map_type = pa.map_(pa.string(), |
| 5307 | pa.list_(pa.struct([("struct_field", pa.binary())]))) |
| 5308 | schema = pa.schema([("maps", map_type)]) |
| 5309 | roundtrip(pd.DataFrame({"maps": [map_of_los, map_of_los, map_of_los]}), |
| 5310 | schema=schema) |
| 5311 | |
| 5312 | |
| 5313 | def test_bytes_column_name_to_pandas(): |