MCPcopy Create free account
hub / github.com/apache/arrow / ShouldSkipPage

Method ShouldSkipPage

cpp/src/parquet/column_reader.cc:382–429  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

380}
381
382bool SerializedPageReader::ShouldSkipPage(EncodedStatistics* data_page_statistics) {
383 const PageType::type page_type = LoadEnumSafe(&current_page_header_.type);
384 if (page_type == PageType::DATA_PAGE) {
385 const format::DataPageHeader& header = current_page_header_.data_page_header;
386 CheckNumValuesInHeader(header.num_values);
387 *data_page_statistics = ExtractStatsFromHeader(header, stats_min_max_field_);
388 seen_num_values_ += header.num_values;
389 if (data_page_filter_) {
390 const EncodedStatistics* filter_statistics =
391 data_page_statistics->is_set() ? data_page_statistics : nullptr;
392 DataPageStats data_page_stats(filter_statistics, header.num_values,
393 /*num_rows=*/std::nullopt);
394 if (data_page_filter_(data_page_stats)) {
395 return true;
396 }
397 }
398 } else if (page_type == PageType::DATA_PAGE_V2) {
399 const format::DataPageHeaderV2& header = current_page_header_.data_page_header_v2;
400 CheckNumValuesInHeader(header.num_values);
401 if (header.num_rows < 0) {
402 throw ParquetException("Invalid page header (negative number of rows)");
403 }
404 if (header.definition_levels_byte_length < 0 ||
405 header.repetition_levels_byte_length < 0) {
406 throw ParquetException("Invalid page header (negative levels byte length)");
407 }
408 *data_page_statistics = ExtractStatsFromHeader(header, stats_min_max_field_);
409 seen_num_values_ += header.num_values;
410 if (data_page_filter_) {
411 const EncodedStatistics* filter_statistics =
412 data_page_statistics->is_set() ? data_page_statistics : nullptr;
413 DataPageStats data_page_stats(filter_statistics, header.num_values,
414 header.num_rows);
415 if (data_page_filter_(data_page_stats)) {
416 return true;
417 }
418 }
419 } else if (page_type == PageType::DICTIONARY_PAGE) {
420 const format::DictionaryPageHeader& dict_header =
421 current_page_header_.dictionary_page_header;
422 CheckNumValuesInHeader(dict_header.num_values);
423 } else {
424 // We don't know what this page type is. We're allowed to skip non-data
425 // pages.
426 return true;
427 }
428 return false;
429}
430
431std::shared_ptr<Page> SerializedPageReader::NextPage() {
432 ThriftDeserializer deserializer(properties_);

Callers

nothing calls this directly

Calls 4

LoadEnumSafeFunction · 0.85
CheckNumValuesInHeaderFunction · 0.85
ExtractStatsFromHeaderFunction · 0.85
ParquetExceptionFunction · 0.85

Tested by

no test coverage detected