MCPcopy Create free account
hub / github.com/apache/arrow / HasCorrectStatistics

Method HasCorrectStatistics

cpp/src/parquet/metadata.cc:1633–1666  ·  view source on GitHub ↗

Reference: parquet-mr/parquet-column/src/main/java/org/apache/parquet/CorruptStatistics.java PARQUET-686 has more discussion on statistics

Source from the content-addressed store, hash-verified

1631// parquet-mr/parquet-column/src/main/java/org/apache/parquet/CorruptStatistics.java
1632// PARQUET-686 has more discussion on statistics
1633bool ApplicationVersion::HasCorrectStatistics(Type::type col_type,
1634 const EncodedStatistics& statistics,
1635 SortOrder::type sort_order) const {
1636 // parquet-cpp version 1.3.0 and parquet-mr 1.10.0 onwards stats are computed
1637 // correctly for all types
1638 if ((application_ == "parquet-cpp" && VersionLt(PARQUET_CPP_FIXED_STATS_VERSION())) ||
1639 (application_ == "parquet-mr" && VersionLt(PARQUET_MR_FIXED_STATS_VERSION()))) {
1640 // Only SIGNED are valid unless max and min are the same
1641 // (in which case the sort order does not matter)
1642 bool max_equals_min = statistics.has_min && statistics.has_max
1643 ? statistics.min() == statistics.max()
1644 : false;
1645 if (SortOrder::SIGNED != sort_order && !max_equals_min) {
1646 return false;
1647 }
1648
1649 // Statistics of other types are OK
1650 if (col_type != Type::FIXED_LEN_BYTE_ARRAY && col_type != Type::BYTE_ARRAY) {
1651 return true;
1652 }
1653 }
1654 // created_by is not populated, which could have been caused by
1655 // parquet-mr during the same time as PARQUET-251, see PARQUET-297
1656 if (application_ == "unknown") {
1657 return true;
1658 }
1659
1660 // PARQUET-251
1661 if (VersionLt(PARQUET_251_FIXED_VERSION())) {
1662 return false;
1663 }
1664
1665 return true;
1666}
1667
1668// MetaData Builders
1669// row-group metadata

Callers 2

is_stats_setMethod · 0.80
TESTFunction · 0.80

Calls 2

minMethod · 0.45
maxMethod · 0.45

Tested by 1

TESTFunction · 0.64