Reference: parquet-mr/parquet-column/src/main/java/org/apache/parquet/CorruptStatistics.java PARQUET-686 has more discussion on statistics
| 1631 | // parquet-mr/parquet-column/src/main/java/org/apache/parquet/CorruptStatistics.java |
| 1632 | // PARQUET-686 has more discussion on statistics |
| 1633 | bool ApplicationVersion::HasCorrectStatistics(Type::type col_type, |
| 1634 | const EncodedStatistics& statistics, |
| 1635 | SortOrder::type sort_order) const { |
| 1636 | // parquet-cpp version 1.3.0 and parquet-mr 1.10.0 onwards stats are computed |
| 1637 | // correctly for all types |
| 1638 | if ((application_ == "parquet-cpp" && VersionLt(PARQUET_CPP_FIXED_STATS_VERSION())) || |
| 1639 | (application_ == "parquet-mr" && VersionLt(PARQUET_MR_FIXED_STATS_VERSION()))) { |
| 1640 | // Only SIGNED are valid unless max and min are the same |
| 1641 | // (in which case the sort order does not matter) |
| 1642 | bool max_equals_min = statistics.has_min && statistics.has_max |
| 1643 | ? statistics.min() == statistics.max() |
| 1644 | : false; |
| 1645 | if (SortOrder::SIGNED != sort_order && !max_equals_min) { |
| 1646 | return false; |
| 1647 | } |
| 1648 | |
| 1649 | // Statistics of other types are OK |
| 1650 | if (col_type != Type::FIXED_LEN_BYTE_ARRAY && col_type != Type::BYTE_ARRAY) { |
| 1651 | return true; |
| 1652 | } |
| 1653 | } |
| 1654 | // created_by is not populated, which could have been caused by |
| 1655 | // parquet-mr during the same time as PARQUET-251, see PARQUET-297 |
| 1656 | if (application_ == "unknown") { |
| 1657 | return true; |
| 1658 | } |
| 1659 | |
| 1660 | // PARQUET-251 |
| 1661 | if (VersionLt(PARQUET_251_FIXED_VERSION())) { |
| 1662 | return false; |
| 1663 | } |
| 1664 | |
| 1665 | return true; |
| 1666 | } |
| 1667 | |
| 1668 | // MetaData Builders |
| 1669 | // row-group metadata |