MCPcopy Create free account
hub / github.com/apache/arrow / test_numpy_binary_overflow_to_chunked

Function test_numpy_binary_overflow_to_chunked

python/pyarrow/tests/test_array.py:3683–3728  ·  view source on GitHub ↗
(large_types)

Source from the content-addressed store, hash-verified

3681@pytest.mark.large_memory
3682@pytest.mark.parametrize('large_types', [False, True])
3683def test_numpy_binary_overflow_to_chunked(large_types):
3684 # ARROW-3762, ARROW-5966, GH-35289
3685
3686 # 2^31 + 1 bytes
3687 values = [b'x']
3688 unicode_values = ['x']
3689
3690 # Make 10 unique 1MB strings then repeat then 2048 times
3691 unique_strings = {
3692 i: b'x' * ((1 << 20) - 1) + str(i % 10).encode('utf8')
3693 for i in range(10)
3694 }
3695 unicode_unique_strings = {i: x.decode('utf8')
3696 for i, x in unique_strings.items()}
3697 values += [unique_strings[i % 10] for i in range(1 << 11)]
3698 unicode_values += [unicode_unique_strings[i % 10]
3699 for i in range(1 << 11)]
3700
3701 binary_type = pa.large_binary() if large_types else pa.binary()
3702 string_type = pa.large_utf8() if large_types else pa.utf8()
3703 for case, ex_type in [(values, binary_type),
3704 (unicode_values, string_type)]:
3705 arr = np.array(case)
3706 arrow_arr = pa.array(arr, ex_type)
3707 arr = None
3708
3709 assert arrow_arr.type == ex_type
3710 if large_types:
3711 # Large types shouldn't be chunked
3712 assert isinstance(arrow_arr, pa.Array)
3713
3714 for i in range(len(arrow_arr)):
3715 val = arrow_arr[i]
3716 assert val.as_py() == case[i]
3717 else:
3718 assert isinstance(arrow_arr, pa.ChunkedArray)
3719
3720 # Split up into 16MB chunks. 128 * 16 = 2048, so 129
3721 assert arrow_arr.num_chunks == 129
3722
3723 value_index = 0
3724 for i in range(arrow_arr.num_chunks):
3725 chunk = arrow_arr.chunk(i)
3726 for val in chunk:
3727 assert val.as_py() == case[value_index]
3728 value_index += 1
3729
3730
3731@pytest.mark.large_memory

Callers

nothing calls this directly

Calls 5

lenFunction · 0.85
as_pyMethod · 0.80
encodeMethod · 0.45
decodeMethod · 0.45
arrayMethod · 0.45

Tested by

no test coverage detected