MCPcopy Create free account
hub / github.com/apache/arrow / PeekImpl

Method PeekImpl

cpp/src/arrow/util/rle_encoding_internal.h:824–897  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

822
823template <typename Handler>
824auto RleBitPackedParser::PeekImpl(Handler&& handler) const
825 -> std::pair<rle_size_t, ControlFlow> {
826 ARROW_DCHECK(!exhausted());
827
828 constexpr auto kMaxSize = bit_util::kMaxLEB128ByteLenFor<uint32_t>;
829 uint32_t run_len_type = 0;
830 const auto header_bytes =
831 bit_util::ParseLeadingLEB128(data_, std::min(kMaxSize, data_size_), &run_len_type);
832 if (ARROW_PREDICT_FALSE(header_bytes == 0)) {
833 // Malformed LEB128 data
834 return {0, ControlFlow::Break};
835 }
836
837 const bool is_bit_packed = run_len_type & 1;
838 const uint32_t count = run_len_type >> 1;
839 if (is_bit_packed) {
840 // Bit-packed run
841 constexpr auto kMaxCount = internal::max_size_for_v<rle_size_t> / 8;
842 if (ARROW_PREDICT_FALSE(count == 0 || count > kMaxCount)) {
843 // Illegal number of encoded values
844 return {0, ControlFlow::Break};
845 }
846
847 ARROW_DCHECK_LT(static_cast<uint64_t>(count) * 8,
848 internal::max_size_for_v<rle_size_t>);
849 // Count Already divided by 8 for byte size calculations
850 auto bytes_read = header_bytes + static_cast<int64_t>(count) * value_bit_width_;
851 auto values_count = static_cast<rle_size_t>(count * 8);
852 if (ARROW_PREDICT_FALSE(bytes_read > data_size_)) {
853 // Bit-packed run would overflow data buffer, but we might still be able
854 // to return a truncated bit-packed such as generated by some non-compliant
855 // encoders.
856 // Example in GH-47981: column contains 25 5-bit values, has a single
857 // bit-packed run with count=4 (theoretically 32 values), but only 17
858 // bytes of RLE-bit-packed data (including the one-byte header).
859 bytes_read = data_size_;
860 values_count =
861 static_cast<rle_size_t>((bytes_read - header_bytes) * 8 / value_bit_width_);
862 // Only allow errors where the bit-packed run is not padded to a multiple
863 // of 8 values. Larger truncation should not occur.
864 if (values_count <= static_cast<rle_size_t>((count - 1) * 8)) {
865 return {0, ControlFlow::Break};
866 }
867 }
868
869 auto control = handler.OnBitPackedRun(
870 BitPackedRun(data_ + header_bytes, values_count, value_bit_width_,
871 /* .max_read_bytes= */ data_size_ - header_bytes));
872
873 return {static_cast<rle_size_t>(bytes_read), control};
874 }
875
876 // RLE run
877 if (ARROW_PREDICT_FALSE(count == 0)) {
878 // Illegal number of encoded values
879 return {0, ControlFlow::Break};
880 }
881

Callers

nothing calls this directly

Calls 4

ParseLeadingLEB128Function · 0.85
BitPackedRunClass · 0.85
BytesForBitsFunction · 0.85
RleRunClass · 0.85

Tested by

no test coverage detected