MCPcopy Create free account
hub / github.com/apache/arrow / BuildDataPageV1

Method BuildDataPageV1

cpp/src/parquet/column_writer.cc:1003–1047  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1001}
1002
1003void ColumnWriterImpl::BuildDataPageV1(int64_t definition_levels_rle_size,
1004 int64_t repetition_levels_rle_size,
1005 int64_t uncompressed_size,
1006 const std::shared_ptr<Buffer>& values) {
1007 // Use Arrow::Buffer::shrink_to_fit = false
1008 // underlying buffer only keeps growing. Resize to a smaller size does not reallocate.
1009 PARQUET_THROW_NOT_OK(uncompressed_data_->Resize(uncompressed_size, false));
1010 ConcatenateBuffers(definition_levels_rle_size, repetition_levels_rle_size, values,
1011 uncompressed_data_->mutable_data());
1012 auto [page_stats, page_size_stats] = GetPageStatistics();
1013 page_stats.ApplyStatSizeLimits(properties_->max_statistics_size(descr_->path()));
1014 page_stats.set_is_signed(SortOrder::SIGNED == descr_->sort_order());
1015 ResetPageStatistics();
1016
1017 std::shared_ptr<Buffer> compressed_data;
1018 if (pager_->has_compressor()) {
1019 pager_->Compress(*(uncompressed_data_.get()), compressor_temp_buffer_.get());
1020 compressed_data = compressor_temp_buffer_;
1021 } else {
1022 compressed_data = uncompressed_data_;
1023 }
1024
1025 int32_t num_values = static_cast<int32_t>(num_buffered_values_);
1026 int64_t first_row_index = rows_written_ - num_buffered_rows_;
1027
1028 // Write the page to OutputStream eagerly if there is no dictionary or
1029 // if dictionary encoding has fallen back to PLAIN
1030 if (has_dictionary_ && !fallback_) { // Save pages until end of dictionary encoding
1031 PARQUET_ASSIGN_OR_THROW(
1032 auto compressed_data_copy,
1033 compressed_data->CopySlice(0, compressed_data->size(), allocator_));
1034 std::unique_ptr<DataPage> page_ptr = std::make_unique<DataPageV1>(
1035 compressed_data_copy, num_values, encoding_, Encoding::RLE, Encoding::RLE,
1036 uncompressed_size, std::move(page_stats), first_row_index,
1037 std::move(page_size_stats));
1038 total_compressed_bytes_ += page_ptr->size() + sizeof(format::PageHeader);
1039
1040 data_pages_.push_back(std::move(page_ptr));
1041 } else { // Eagerly write pages
1042 DataPageV1 page(compressed_data, num_values, encoding_, Encoding::RLE, Encoding::RLE,
1043 uncompressed_size, std::move(page_stats), first_row_index,
1044 std::move(page_size_stats));
1045 WriteDataPage(page);
1046 }
1047}
1048
1049void ColumnWriterImpl::BuildDataPageV2(int64_t definition_levels_rle_size,
1050 int64_t repetition_levels_rle_size,

Callers

nothing calls this directly

Calls 10

ConcatenateBuffersFunction · 0.85
push_backMethod · 0.80
ResizeMethod · 0.45
mutable_dataMethod · 0.45
pathMethod · 0.45
sort_orderMethod · 0.45
has_compressorMethod · 0.45
CompressMethod · 0.45
getMethod · 0.45
sizeMethod · 0.45

Tested by

no test coverage detected