Reference: parquet-mr/parquet-column/src/main/java/org/apache/parquet/CorruptStatistics.java PARQUET-686 has more discussion on statistics
| 1605 | // parquet-mr/parquet-column/src/main/java/org/apache/parquet/CorruptStatistics.java |
| 1606 | // PARQUET-686 has more discussion on statistics |
| 1607 | bool ApplicationVersion::HasCorrectStatistics(Type::type col_type, |
| 1608 | const EncodedStatistics& statistics, |
| 1609 | SortOrder::type sort_order) const { |
| 1610 | // parquet-cpp version 1.3.0 and parquet-mr 1.10.0 onwards stats are computed |
| 1611 | // correctly for all types |
| 1612 | if ((application_ == "parquet-cpp" && VersionLt(PARQUET_CPP_FIXED_STATS_VERSION())) || |
| 1613 | (application_ == "parquet-mr" && VersionLt(PARQUET_MR_FIXED_STATS_VERSION()))) { |
| 1614 | // Only SIGNED are valid unless max and min are the same |
| 1615 | // (in which case the sort order does not matter) |
| 1616 | bool max_equals_min = statistics.has_min && statistics.has_max |
| 1617 | ? statistics.min() == statistics.max() |
| 1618 | : false; |
| 1619 | if (SortOrder::SIGNED != sort_order && !max_equals_min) { |
| 1620 | return false; |
| 1621 | } |
| 1622 | |
| 1623 | // Statistics of other types are OK |
| 1624 | if (col_type != Type::FIXED_LEN_BYTE_ARRAY && col_type != Type::BYTE_ARRAY) { |
| 1625 | return true; |
| 1626 | } |
| 1627 | } |
| 1628 | // created_by is not populated, which could have been caused by |
| 1629 | // parquet-mr during the same time as PARQUET-251, see PARQUET-297 |
| 1630 | if (application_ == "unknown") { |
| 1631 | return true; |
| 1632 | } |
| 1633 | |
| 1634 | // PARQUET-251 |
| 1635 | if (VersionLt(PARQUET_251_FIXED_VERSION())) { |
| 1636 | return false; |
| 1637 | } |
| 1638 | |
| 1639 | return true; |
| 1640 | } |
| 1641 | |
| 1642 | // MetaData Builders |
| 1643 | // row-group metadata |