MCPcopy Create free account
hub / github.com/apache/arrow / CompressBuffer

Method CompressBuffer

cpp/src/arrow/ipc/writer.cc:196–236  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

194 }
195
196 Status CompressBuffer(const Buffer& buffer, util::Codec* codec,
197 std::shared_ptr<Buffer>* out) {
198 // Convert buffer to uncompressed-length-prefixed buffer. The actual body may or may
199 // not be compressed, depending on user-preference and projected size reduction.
200 int64_t maximum_length = codec->MaxCompressedLen(buffer.size(), buffer.data());
201 int64_t prefixed_length = buffer.size();
202
203 ARROW_ASSIGN_OR_RAISE(
204 auto result,
205 AllocateResizableBuffer(maximum_length + sizeof(int64_t), options_.memory_pool));
206 ARROW_ASSIGN_OR_RAISE(auto actual_length,
207 codec->Compress(buffer.size(), buffer.data(), maximum_length,
208 result->mutable_data() + sizeof(int64_t)));
209 // FIXME: Not the most sophisticated way to handle this. Ideally, you'd want to avoid
210 // pre-compressing the entire buffer via some kind of sampling method. As the feature
211 // gains adoption, this may become a worthwhile optimization.
212 //
213 // See: GH-33885
214 if (!ShouldCompress(buffer.size(), actual_length)) {
215 if (buffer.size() < actual_length || buffer.size() > maximum_length) {
216 RETURN_NOT_OK(
217 result->Resize(buffer.size() + sizeof(int64_t), /*shrink_to_fit=*/false));
218 result->ZeroPadding();
219 }
220 std::memcpy(result->mutable_data() + sizeof(int64_t), buffer.data(),
221 static_cast<size_t>(buffer.size()));
222 actual_length = buffer.size();
223 // Size of -1 indicates to the reader that the body doesn't need to be decompressed
224 prefixed_length = -1;
225 } else {
226 // Shrink compressed buffer
227 RETURN_NOT_OK(
228 result->Resize(actual_length + sizeof(int64_t), /* shrink_to_fit= */ true));
229 }
230 int64_t prefixed_length_little_endian = bit_util::ToLittleEndian(prefixed_length);
231 util::SafeStore(result->mutable_data(), prefixed_length_little_endian);
232
233 *out = SliceBuffer(std::move(result), /*offset=*/0, actual_length + sizeof(int64_t));
234
235 return Status::OK();
236 }
237
238 Status CompressBodyBuffers() {
239 RETURN_NOT_OK(

Callers

nothing calls this directly

Calls 9

SafeStoreFunction · 0.85
SliceBufferFunction · 0.85
ToLittleEndianFunction · 0.50
OKFunction · 0.50
MaxCompressedLenMethod · 0.45
sizeMethod · 0.45
dataMethod · 0.45
ResizeMethod · 0.45
mutable_dataMethod · 0.45

Tested by

no test coverage detected