| 429 | } |
| 430 | |
| 431 | std::shared_ptr<Page> SerializedPageReader::NextPage() { |
| 432 | ThriftDeserializer deserializer(properties_); |
| 433 | |
| 434 | // Loop here because there may be unhandled page types that we skip until |
| 435 | // finding a page that we do know what to do with |
| 436 | while (seen_num_values_ < total_num_values_) { |
| 437 | int64_t header_size = 0; |
| 438 | int64_t allowed_page_size = kDefaultPageHeaderSize; |
| 439 | |
| 440 | // Page headers can be very large because of page statistics |
| 441 | // We try to deserialize a larger buffer progressively |
| 442 | // until a maximum allowed header limit |
| 443 | while (true) { |
| 444 | PARQUET_ASSIGN_OR_THROW(auto view, stream_->Peek(allowed_page_size)); |
| 445 | if (view.size() == 0) return nullptr; |
| 446 | |
| 447 | try { |
| 448 | if (meta_decryptor_ != nullptr) { |
| 449 | UpdateDecryption(meta_decryptor_.get(), encryption::kDictionaryPageHeader, |
| 450 | &data_page_header_aad_); |
| 451 | } |
| 452 | // Reset current page header to avoid unclearing the __isset flag. |
| 453 | current_page_header_ = format::PageHeader(); |
| 454 | header_size = deserializer.DeserializeMessage( |
| 455 | reinterpret_cast<const uint8_t*>(view.data()), view.size(), |
| 456 | ¤t_page_header_, meta_decryptor_.get()); |
| 457 | break; |
| 458 | } catch (std::exception& e) { |
| 459 | // Failed to deserialize. Double the allowed page header size and try again |
| 460 | std::stringstream ss; |
| 461 | ss << e.what(); |
| 462 | allowed_page_size *= 2; |
| 463 | if (allowed_page_size > max_page_header_size_) { |
| 464 | ss << "Deserializing page header failed.\n"; |
| 465 | throw ParquetException(ss.str()); |
| 466 | } |
| 467 | } |
| 468 | } |
| 469 | // Advance the stream offset |
| 470 | PARQUET_THROW_NOT_OK(stream_->Advance(header_size)); |
| 471 | |
| 472 | int32_t compressed_len = current_page_header_.compressed_page_size; |
| 473 | int32_t uncompressed_len = current_page_header_.uncompressed_page_size; |
| 474 | if (compressed_len < 0 || uncompressed_len < 0) { |
| 475 | throw ParquetException("Invalid page header"); |
| 476 | } |
| 477 | |
| 478 | EncodedStatistics data_page_statistics; |
| 479 | if (ShouldSkipPage(&data_page_statistics)) { |
| 480 | PARQUET_THROW_NOT_OK(stream_->Advance(compressed_len)); |
| 481 | continue; |
| 482 | } |
| 483 | |
| 484 | if (data_decryptor_ != nullptr) { |
| 485 | UpdateDecryption(data_decryptor_.get(), encryption::kDictionaryPage, |
| 486 | &data_page_aad_); |
| 487 | } |
| 488 | |