Method BuildDataPageV2

cpp/src/parquet/column_writer.cc:1051–1109

}

void ColumnWriterImpl::BuildDataPageV2(int64_t definition_levels_rle_size,
                                       int64_t repetition_levels_rle_size,
                                       int64_t uncompressed_size,
                                       const std::shared_ptr<Buffer>& values) {
  // Compress the values if needed. Repetition and definition levels are uncompressed in
  // V2.
  bool page_is_compressed = false;
  if (pager_->has_compressor() && values->size() > 0) {
    pager_->Compress(*values, compressor_temp_buffer_.get());
    if (compressor_temp_buffer_->size() < values->size()) {
      page_is_compressed = true;
    }
  }
  std::shared_ptr<Buffer> compressed_values =
      (page_is_compressed ? compressor_temp_buffer_ : values);

  // Concatenate uncompressed levels and the possibly compressed values
  int64_t combined_size =
      definition_levels_rle_size + repetition_levels_rle_size + compressed_values->size();
  std::shared_ptr<ResizableBuffer> combined = AllocateBuffer(allocator_, combined_size);

  ConcatenateBuffers(definition_levels_rle_size, repetition_levels_rle_size,
                     compressed_values, combined->mutable_data());

  auto [page_stats, page_size_stats] = GetPageStatistics();
  page_stats.ApplyStatSizeLimits(properties_->max_statistics_size(descr_->path()));
  page_stats.set_is_signed(SortOrder::SIGNED == descr_->sort_order());
  ResetPageStatistics();

  int32_t num_values = static_cast<int32_t>(num_buffered_values_);
  int32_t null_count = static_cast<int32_t>(num_buffered_nulls_);
  int32_t num_rows = static_cast<int32_t>(num_buffered_rows_);
  int32_t def_levels_byte_length = static_cast<int32_t>(definition_levels_rle_size);
  int32_t rep_levels_byte_length = static_cast<int32_t>(repetition_levels_rle_size);
  int64_t first_row_index = rows_written_ - num_buffered_rows_;

  // page_stats.null_count is not set when page_statistics_ is nullptr. It is only used
  // here for safety check.
  DCHECK(!page_stats.has_null_count || page_stats.null_count == null_count);

  // Write the page to OutputStream eagerly if there is no dictionary or
  // if dictionary encoding has fallen back to PLAIN
  if (has_dictionary_ && !fallback_) {  // Save pages until end of dictionary encoding
    PARQUET_ASSIGN_OR_THROW(auto data_copy,
                            combined->CopySlice(0, combined->size(), allocator_));
    std::unique_ptr<DataPage> page_ptr = std::make_unique<DataPageV2>(
        combined, num_values, null_count, num_rows, encoding_, def_levels_byte_length,
        rep_levels_byte_length, uncompressed_size, page_is_compressed,
        std::move(page_stats), first_row_index, std::move(page_size_stats));
    total_compressed_bytes_ += page_ptr->size() + sizeof(format::PageHeader);
    data_pages_.push_back(std::move(page_ptr));
  } else {
    DataPageV2 page(combined, num_values, null_count, num_rows, encoding_,
                    def_levels_byte_length, rep_levels_byte_length, uncompressed_size,
                    page_is_compressed, std::move(page_stats), first_row_index,
                    std::move(page_size_stats));
    WriteDataPage(page);
  }