MCPcopy Create free account
hub / github.com/apache/arrow / BuildDataPageV2

Method BuildDataPageV2

cpp/src/parquet/column_writer.cc:1049–1107  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1047}
1048
1049void ColumnWriterImpl::BuildDataPageV2(int64_t definition_levels_rle_size,
1050 int64_t repetition_levels_rle_size,
1051 int64_t uncompressed_size,
1052 const std::shared_ptr<Buffer>& values) {
1053 // Compress the values if needed. Repetition and definition levels are uncompressed in
1054 // V2.
1055 bool page_is_compressed = false;
1056 if (pager_->has_compressor() && values->size() > 0) {
1057 pager_->Compress(*values, compressor_temp_buffer_.get());
1058 if (compressor_temp_buffer_->size() < values->size()) {
1059 page_is_compressed = true;
1060 }
1061 }
1062 std::shared_ptr<Buffer> compressed_values =
1063 (page_is_compressed ? compressor_temp_buffer_ : values);
1064
1065 // Concatenate uncompressed levels and the possibly compressed values
1066 int64_t combined_size =
1067 definition_levels_rle_size + repetition_levels_rle_size + compressed_values->size();
1068 std::shared_ptr<ResizableBuffer> combined = AllocateBuffer(allocator_, combined_size);
1069
1070 ConcatenateBuffers(definition_levels_rle_size, repetition_levels_rle_size,
1071 compressed_values, combined->mutable_data());
1072
1073 auto [page_stats, page_size_stats] = GetPageStatistics();
1074 page_stats.ApplyStatSizeLimits(properties_->max_statistics_size(descr_->path()));
1075 page_stats.set_is_signed(SortOrder::SIGNED == descr_->sort_order());
1076 ResetPageStatistics();
1077
1078 int32_t num_values = static_cast<int32_t>(num_buffered_values_);
1079 int32_t null_count = static_cast<int32_t>(num_buffered_nulls_);
1080 int32_t num_rows = static_cast<int32_t>(num_buffered_rows_);
1081 int32_t def_levels_byte_length = static_cast<int32_t>(definition_levels_rle_size);
1082 int32_t rep_levels_byte_length = static_cast<int32_t>(repetition_levels_rle_size);
1083 int64_t first_row_index = rows_written_ - num_buffered_rows_;
1084
1085 // page_stats.null_count is not set when page_statistics_ is nullptr. It is only used
1086 // here for safety check.
1087 DCHECK(!page_stats.has_null_count || page_stats.null_count == null_count);
1088
1089 // Write the page to OutputStream eagerly if there is no dictionary or
1090 // if dictionary encoding has fallen back to PLAIN
1091 if (has_dictionary_ && !fallback_) { // Save pages until end of dictionary encoding
1092 PARQUET_ASSIGN_OR_THROW(auto data_copy,
1093 combined->CopySlice(0, combined->size(), allocator_));
1094 std::unique_ptr<DataPage> page_ptr = std::make_unique<DataPageV2>(
1095 combined, num_values, null_count, num_rows, encoding_, def_levels_byte_length,
1096 rep_levels_byte_length, uncompressed_size, page_is_compressed,
1097 std::move(page_stats), first_row_index, std::move(page_size_stats));
1098 total_compressed_bytes_ += page_ptr->size() + sizeof(format::PageHeader);
1099 data_pages_.push_back(std::move(page_ptr));
1100 } else {
1101 DataPageV2 page(combined, num_values, null_count, num_rows, encoding_,
1102 def_levels_byte_length, rep_levels_byte_length, uncompressed_size,
1103 page_is_compressed, std::move(page_stats), first_row_index,
1104 std::move(page_size_stats));
1105 WriteDataPage(page);
1106 }

Callers

nothing calls this directly

Calls 10

ConcatenateBuffersFunction · 0.85
push_backMethod · 0.80
AllocateBufferFunction · 0.70
has_compressorMethod · 0.45
sizeMethod · 0.45
CompressMethod · 0.45
getMethod · 0.45
mutable_dataMethod · 0.45
pathMethod · 0.45
sort_orderMethod · 0.45

Tested by

no test coverage detected