MCPcopy Create free account
hub / github.com/apache/arrow / ConvertPySequence

Function ConvertPySequence

python/pyarrow/src/arrow/python/python_to_arrow.cc:1267–1333  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1265} // namespace
1266
1267Result<std::shared_ptr<ChunkedArray>> ConvertPySequence(PyObject* obj, PyObject* mask,
1268 PyConversionOptions options,
1269 MemoryPool* pool) {
1270 PyAcquireGIL lock;
1271
1272 PyObject* seq = nullptr;
1273 OwnedRef tmp_seq_nanny;
1274
1275 ARROW_ASSIGN_OR_RAISE(auto is_pandas_imported, internal::IsModuleImported("pandas"));
1276 if (is_pandas_imported) {
1277 // If pandas has been already imported initialize the static pandas objects to
1278 // support converting from pd.Timedelta and pd.Timestamp objects
1279 internal::InitPandasStaticData();
1280 }
1281
1282 int64_t size = options.size;
1283 RETURN_NOT_OK(ConvertToSequenceAndInferSize(obj, &seq, &size));
1284 tmp_seq_nanny.reset(seq);
1285
1286 // In some cases, type inference may be "loose", like strings. If the user
1287 // passed pa.string(), then we will error if we encounter any non-UTF8
1288 // value. If not, then we will allow the result to be a BinaryArray
1289 std::shared_ptr<DataType> extension_type;
1290 if (options.type == nullptr) {
1291 ARROW_ASSIGN_OR_RAISE(options.type, InferArrowType(seq, mask, options.from_pandas));
1292 options.strict = false;
1293 // If type inference returned an extension type, convert using
1294 // the storage type and then wrap the result as an extension array
1295 if (options.type->id() == Type::EXTENSION) {
1296 extension_type = options.type;
1297 options.type = checked_cast<const ExtensionType&>(*options.type).storage_type();
1298 }
1299 } else {
1300 options.strict = true;
1301 }
1302 ARROW_DCHECK_GE(size, 0);
1303
1304 ARROW_ASSIGN_OR_RAISE(auto converter, (MakeConverter<PyConverter, PyConverterTrait>(
1305 options.type, options, pool)));
1306 std::shared_ptr<ChunkedArray> result;
1307 if (converter->may_overflow()) {
1308 // The converter hierarchy contains binary- or list-like builders which can overflow
1309 // depending on the input values. Wrap the converter with a chunker which detects
1310 // the overflow and automatically creates new chunks.
1311 ARROW_ASSIGN_OR_RAISE(auto chunked_converter, MakeChunker(std::move(converter)));
1312 if (mask != nullptr && mask != Py_None) {
1313 RETURN_NOT_OK(chunked_converter->ExtendMasked(seq, mask, size));
1314 } else {
1315 RETURN_NOT_OK(chunked_converter->Extend(seq, size));
1316 }
1317 ARROW_ASSIGN_OR_RAISE(result, chunked_converter->ToChunkedArray());
1318 } else {
1319 // If the converter can't overflow spare the capacity error checking on the hot-path,
1320 // this improves the performance roughly by ~10% for primitive types.
1321 if (mask != nullptr && mask != Py_None) {
1322 RETURN_NOT_OK(converter->ExtendMasked(seq, mask, size));
1323 } else {
1324 RETURN_NOT_OK(converter->Extend(seq, size));

Callers 4

TestMixedTypeFailsFunction · 0.85
TestNoneAndNaNFunction · 0.85

Calls 11

storage_typeMethod · 0.80
may_overflowMethod · 0.80
InferArrowTypeFunction · 0.70
ARROW_ASSIGN_OR_RAISEFunction · 0.50
MakeChunkerFunction · 0.50
resetMethod · 0.45
idMethod · 0.45
ExtendMaskedMethod · 0.45
ExtendMethod · 0.45
ToChunkedArrayMethod · 0.45

Tested by 4

TestMixedTypeFailsFunction · 0.68
TestNoneAndNaNFunction · 0.68