MCPcopy Create free account
hub / github.com/apache/arrow / ConvertPySequence

Function ConvertPySequence

python/pyarrow/src/arrow/python/python_to_arrow.cc:1284–1350  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1282} // namespace
1283
1284Result<std::shared_ptr<ChunkedArray>> ConvertPySequence(PyObject* obj, PyObject* mask,
1285 PyConversionOptions options,
1286 MemoryPool* pool) {
1287 PyAcquireGIL lock;
1288
1289 PyObject* seq = nullptr;
1290 OwnedRef tmp_seq_nanny;
1291
1292 ARROW_ASSIGN_OR_RAISE(auto is_pandas_imported, internal::IsModuleImported("pandas"));
1293 if (is_pandas_imported) {
1294 // If pandas has been already imported initialize the static pandas objects to
1295 // support converting from pd.Timedelta and pd.Timestamp objects
1296 internal::InitPandasStaticData();
1297 }
1298
1299 int64_t size = options.size;
1300 RETURN_NOT_OK(ConvertToSequenceAndInferSize(obj, &seq, &size));
1301 tmp_seq_nanny.reset(seq);
1302
1303 // In some cases, type inference may be "loose", like strings. If the user
1304 // passed pa.string(), then we will error if we encounter any non-UTF8
1305 // value. If not, then we will allow the result to be a BinaryArray
1306 std::shared_ptr<DataType> extension_type;
1307 if (options.type == nullptr) {
1308 ARROW_ASSIGN_OR_RAISE(options.type, InferArrowType(seq, mask, options.from_pandas));
1309 options.strict = false;
1310 // If type inference returned an extension type, convert using
1311 // the storage type and then wrap the result as an extension array
1312 if (options.type->id() == Type::EXTENSION) {
1313 extension_type = options.type;
1314 options.type = checked_cast<const ExtensionType&>(*options.type).storage_type();
1315 }
1316 } else {
1317 options.strict = true;
1318 }
1319 ARROW_DCHECK_GE(size, 0);
1320
1321 ARROW_ASSIGN_OR_RAISE(auto converter, (MakeConverter<PyConverter, PyConverterTrait>(
1322 options.type, options, pool)));
1323 std::shared_ptr<ChunkedArray> result;
1324 if (converter->may_overflow()) {
1325 // The converter hierarchy contains binary- or list-like builders which can overflow
1326 // depending on the input values. Wrap the converter with a chunker which detects
1327 // the overflow and automatically creates new chunks.
1328 ARROW_ASSIGN_OR_RAISE(auto chunked_converter, MakeChunker(std::move(converter)));
1329 if (mask != nullptr && mask != Py_None) {
1330 RETURN_NOT_OK(chunked_converter->ExtendMasked(seq, mask, size));
1331 } else {
1332 RETURN_NOT_OK(chunked_converter->Extend(seq, size));
1333 }
1334 ARROW_ASSIGN_OR_RAISE(result, chunked_converter->ToChunkedArray());
1335 } else {
1336 // If the converter can't overflow spare the capacity error checking on the hot-path,
1337 // this improves the performance roughly by ~10% for primitive types.
1338 if (mask != nullptr && mask != Py_None) {
1339 RETURN_NOT_OK(converter->ExtendMasked(seq, mask, size));
1340 } else {
1341 RETURN_NOT_OK(converter->Extend(seq, size));

Callers 4

TestMixedTypeFailsFunction · 0.85
TestNoneAndNaNFunction · 0.85

Calls 11

storage_typeMethod · 0.80
may_overflowMethod · 0.80
InferArrowTypeFunction · 0.70
ARROW_ASSIGN_OR_RAISEFunction · 0.50
MakeChunkerFunction · 0.50
resetMethod · 0.45
idMethod · 0.45
ExtendMaskedMethod · 0.45
ExtendMethod · 0.45
ToChunkedArrayMethod · 0.45

Tested by 4

TestMixedTypeFailsFunction · 0.68
TestNoneAndNaNFunction · 0.68