| 3681 | @pytest.mark.large_memory |
| 3682 | @pytest.mark.parametrize('large_types', [False, True]) |
| 3683 | def test_numpy_binary_overflow_to_chunked(large_types): |
| 3684 | # ARROW-3762, ARROW-5966, GH-35289 |
| 3685 | |
| 3686 | # 2^31 + 1 bytes |
| 3687 | values = [b'x'] |
| 3688 | unicode_values = ['x'] |
| 3689 | |
| 3690 | # Make 10 unique 1MB strings then repeat then 2048 times |
| 3691 | unique_strings = { |
| 3692 | i: b'x' * ((1 << 20) - 1) + str(i % 10).encode('utf8') |
| 3693 | for i in range(10) |
| 3694 | } |
| 3695 | unicode_unique_strings = {i: x.decode('utf8') |
| 3696 | for i, x in unique_strings.items()} |
| 3697 | values += [unique_strings[i % 10] for i in range(1 << 11)] |
| 3698 | unicode_values += [unicode_unique_strings[i % 10] |
| 3699 | for i in range(1 << 11)] |
| 3700 | |
| 3701 | binary_type = pa.large_binary() if large_types else pa.binary() |
| 3702 | string_type = pa.large_utf8() if large_types else pa.utf8() |
| 3703 | for case, ex_type in [(values, binary_type), |
| 3704 | (unicode_values, string_type)]: |
| 3705 | arr = np.array(case) |
| 3706 | arrow_arr = pa.array(arr, ex_type) |
| 3707 | arr = None |
| 3708 | |
| 3709 | assert arrow_arr.type == ex_type |
| 3710 | if large_types: |
| 3711 | # Large types shouldn't be chunked |
| 3712 | assert isinstance(arrow_arr, pa.Array) |
| 3713 | |
| 3714 | for i in range(len(arrow_arr)): |
| 3715 | val = arrow_arr[i] |
| 3716 | assert val.as_py() == case[i] |
| 3717 | else: |
| 3718 | assert isinstance(arrow_arr, pa.ChunkedArray) |
| 3719 | |
| 3720 | # Split up into 16MB chunks. 128 * 16 = 2048, so 129 |
| 3721 | assert arrow_arr.num_chunks == 129 |
| 3722 | |
| 3723 | value_index = 0 |
| 3724 | for i in range(arrow_arr.num_chunks): |
| 3725 | chunk = arrow_arr.chunk(i) |
| 3726 | for val in chunk: |
| 3727 | assert val.as_py() == case[value_index] |
| 3728 | value_index += 1 |
| 3729 | |
| 3730 | |
| 3731 | @pytest.mark.large_memory |