MCPcopy Create free account
hub / github.com/apache/arrow / ConvertListsLike

Function ConvertListsLike

python/pyarrow/src/arrow/python/arrow_to_pandas.cc:770–845  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

768
769template <typename T>
770enable_if_list_like<T, Status> ConvertListsLike(PandasOptions options,
771 const ChunkedArray& data,
772 PyObject** out_values) {
773 using ListArrayT = typename TypeTraits<T>::ArrayType;
774 // Get column of underlying value arrays
775 ArrayVector value_arrays;
776 for (int c = 0; c < data.num_chunks(); c++) {
777 const auto& arr = checked_cast<const ListArrayT&>(*data.chunk(c));
778 // values() does not account for offsets, so we need to slice into it.
779 // We can't use Flatten(), because it removes the values behind a null list
780 // value, and that makes the offsets into original list values and our
781 // flattened_values array different.
782 std::shared_ptr<Array> flattened_values = arr.values()->Slice(
783 arr.value_offset(0), arr.value_offset(arr.length()) - arr.value_offset(0));
784 if (arr.value_type()->id() == Type::EXTENSION) {
785 const auto& arr_ext = checked_cast<const ExtensionArray&>(*flattened_values);
786 value_arrays.emplace_back(arr_ext.storage());
787 } else {
788 value_arrays.emplace_back(flattened_values);
789 }
790 }
791
792 using ListArrayType = typename ListArrayT::TypeClass;
793 const auto& list_type = checked_cast<const ListArrayType&>(*data.type());
794 auto value_type = list_type.value_type();
795 if (value_type->id() == Type::EXTENSION) {
796 value_type = checked_cast<const ExtensionType&>(*value_type).storage_type();
797 }
798
799 auto flat_column = std::make_shared<ChunkedArray>(value_arrays, value_type);
800
801 options = MakeInnerOptions(std::move(options));
802
803 OwnedRefNoGIL owned_numpy_array;
804 RETURN_NOT_OK(ConvertChunkedArrayToPandas(options, flat_column, nullptr,
805 owned_numpy_array.ref()));
806 PyObject* numpy_array = owned_numpy_array.obj();
807 ARROW_DCHECK(PyArray_Check(numpy_array));
808
809 int64_t chunk_offset = 0;
810 for (int c = 0; c < data.num_chunks(); c++) {
811 const auto& arr = checked_cast<const ListArrayT&>(*data.chunk(c));
812 const bool has_nulls = data.null_count() > 0;
813 for (int64_t i = 0; i < arr.length(); ++i) {
814 if (has_nulls && arr.IsNull(i)) {
815 Py_INCREF(Py_None);
816 *out_values = Py_None;
817 } else {
818 // Need to subtract value_offset(0) since the original chunk might be a slice
819 // into another array.
820 OwnedRef start(PyLong_FromLongLong(arr.value_offset(i) + chunk_offset -
821 arr.value_offset(0)));
822 OwnedRef end(PyLong_FromLongLong(arr.value_offset(i + 1) + chunk_offset -
823 arr.value_offset(0)));
824 OwnedRef slice(PySlice_New(start.obj(), end.obj(), nullptr));
825
826 if (ARROW_PREDICT_FALSE(slice.obj() == nullptr)) {
827 // Fall out of loop, will return from RETURN_IF_PYERROR

Callers

nothing calls this directly

Calls 15

MakeInnerOptionsFunction · 0.85
emplace_backMethod · 0.80
storage_typeMethod · 0.80
OKFunction · 0.50
num_chunksMethod · 0.45
SliceMethod · 0.45
valuesMethod · 0.45
value_offsetMethod · 0.45
lengthMethod · 0.45
idMethod · 0.45
value_typeMethod · 0.45

Tested by

no test coverage detected