MCPcopy Create free account
hub / github.com/apache/arrow / NextPage

Method NextPage

cpp/src/parquet/column_reader.cc:431–579  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

429}
430
431std::shared_ptr<Page> SerializedPageReader::NextPage() {
432 ThriftDeserializer deserializer(properties_);
433
434 // Loop here because there may be unhandled page types that we skip until
435 // finding a page that we do know what to do with
436 while (seen_num_values_ < total_num_values_) {
437 int64_t header_size = 0;
438 int64_t allowed_page_size = kDefaultPageHeaderSize;
439
440 // Page headers can be very large because of page statistics
441 // We try to deserialize a larger buffer progressively
442 // until a maximum allowed header limit
443 while (true) {
444 PARQUET_ASSIGN_OR_THROW(auto view, stream_->Peek(allowed_page_size));
445 if (view.size() == 0) return nullptr;
446
447 try {
448 if (meta_decryptor_ != nullptr) {
449 UpdateDecryption(meta_decryptor_.get(), encryption::kDictionaryPageHeader,
450 &data_page_header_aad_);
451 }
452 // Reset current page header to avoid unclearing the __isset flag.
453 current_page_header_ = format::PageHeader();
454 header_size = deserializer.DeserializeMessage(
455 reinterpret_cast<const uint8_t*>(view.data()), view.size(),
456 &current_page_header_, meta_decryptor_.get());
457 break;
458 } catch (std::exception& e) {
459 // Failed to deserialize. Double the allowed page header size and try again
460 std::stringstream ss;
461 ss << e.what();
462 allowed_page_size *= 2;
463 if (allowed_page_size > max_page_header_size_) {
464 ss << "Deserializing page header failed.\n";
465 throw ParquetException(ss.str());
466 }
467 }
468 }
469 // Advance the stream offset
470 PARQUET_THROW_NOT_OK(stream_->Advance(header_size));
471
472 int32_t compressed_len = current_page_header_.compressed_page_size;
473 int32_t uncompressed_len = current_page_header_.uncompressed_page_size;
474 if (compressed_len < 0 || uncompressed_len < 0) {
475 throw ParquetException("Invalid page header");
476 }
477
478 EncodedStatistics data_page_statistics;
479 if (ShouldSkipPage(&data_page_statistics)) {
480 PARQUET_THROW_NOT_OK(stream_->Advance(compressed_len));
481 continue;
482 }
483
484 if (data_decryptor_ != nullptr) {
485 UpdateDecryption(data_decryptor_.get(), encryption::kDictionaryPage,
486 &data_page_aad_);
487 }
488

Callers 15

CheckCorrectCrcMethod · 0.45
CheckCorrectDictCrcMethod · 0.45
CheckNextPageCorruptMethod · 0.45
TEST_FFunction · 0.45
TESTFunction · 0.45
TESTFunction · 0.45
ASSERT_OK_AND_ASSIGNFunction · 0.45
GetColumnParquetInfoFunction · 0.45
ReadNewPageMethod · 0.45
TestPageSerdeCrcMethod · 0.45
TEST_FFunction · 0.45
TYPED_TESTFunction · 0.45

Calls 15

PageHeaderClass · 0.85
ParquetExceptionFunction · 0.85
LoadEnumSafeFunction · 0.85
PageCanUseChecksumFunction · 0.85
crc32Function · 0.85
to_stringFunction · 0.85
AddWithOverflowFunction · 0.85
DeserializeMessageMethod · 0.80
strMethod · 0.80
AllocateBufferFunction · 0.70
sizeMethod · 0.45

Tested by 15

CheckCorrectCrcMethod · 0.36
CheckCorrectDictCrcMethod · 0.36
CheckNextPageCorruptMethod · 0.36
TEST_FFunction · 0.36
TESTFunction · 0.36
TESTFunction · 0.36
ASSERT_OK_AND_ASSIGNFunction · 0.36
GetColumnParquetInfoFunction · 0.36
TestPageSerdeCrcMethod · 0.36
TEST_FFunction · 0.36
TYPED_TESTFunction · 0.36