MCPcopy Create free account
hub / github.com/apache/arrow / AttachStatistics

Function AttachStatistics

cpp/src/parquet/arrow/reader_internal.cc:368–422  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

366
367template <typename ArrowType, typename ParquetType>
368void AttachStatistics(::arrow::ArrayData* data,
369 std::unique_ptr<::parquet::ColumnChunkMetaData> metadata,
370 const ReaderContext* ctx) {
371 if (!metadata) {
372 return;
373 }
374
375 using ArrowCType = typename ArrowType::c_type;
376
377 auto statistics = metadata->statistics();
378 if (data->null_count == ::arrow::kUnknownNullCount && !statistics) {
379 return;
380 }
381
382 auto array_statistics = std::make_shared<::arrow::ArrayStatistics>();
383 if (data->null_count != ::arrow::kUnknownNullCount) {
384 array_statistics->null_count = data->null_count;
385 }
386 if (statistics) {
387 if (statistics->HasDistinctCount()) {
388 array_statistics->distinct_count = statistics->distinct_count();
389 }
390 if (statistics->HasMinMax()) {
391 const auto* typed_statistics =
392 checked_cast<const ::parquet::TypedStatistics<ParquetType>*>(statistics.get());
393 const ArrowCType min = typed_statistics->min();
394 const ArrowCType max = typed_statistics->max();
395 if constexpr (std::is_same_v<ArrowCType, bool>) {
396 array_statistics->min = static_cast<bool>(min);
397 array_statistics->max = static_cast<bool>(max);
398 } else if constexpr (std::is_floating_point_v<ArrowCType>) {
399 array_statistics->min = static_cast<double>(min);
400 array_statistics->max = static_cast<double>(max);
401 } else if constexpr (std::is_signed_v<ArrowCType>) {
402 array_statistics->min = static_cast<int64_t>(min);
403 array_statistics->max = static_cast<int64_t>(max);
404 } else {
405 array_statistics->min = static_cast<uint64_t>(min);
406 array_statistics->max = static_cast<uint64_t>(max);
407 }
408 // We can assume that integer/floating point number/boolean
409 // based min/max are always exact if they exist. Apache
410 // Parquet's "Statistics" has "is_min_value_exact" and
411 // "is_max_value_exact" but we can ignore them for integer/
412 // floating point number/boolean based min/max.
413 //
414 // See also the discussion at dev@parquet.apache.org:
415 // https://lists.apache.org/thread/zfnmg5p51b7oylft5w5k4670wgkd4zv4
416 array_statistics->is_min_exact = true;
417 array_statistics->is_max_exact = true;
418 }
419 }
420
421 data->statistics = std::move(array_statistics);
422}
423
424template <typename ArrowType, typename ParquetType>
425Status TransferInt(RecordReader* reader,

Callers

nothing calls this directly

Calls 7

HasDistinctCountMethod · 0.80
HasMinMaxMethod · 0.80
statisticsMethod · 0.45
distinct_countMethod · 0.45
getMethod · 0.45
minMethod · 0.45
maxMethod · 0.45

Tested by

no test coverage detected