MCPcopy Create free account
hub / github.com/apache/arrow / DecodeArrowDense

Method DecodeArrowDense

cpp/src/parquet/decoder.cc:1313–1368  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1311
1312 private:
1313 Status DecodeArrowDense(int num_values, int null_count, const uint8_t* valid_bits,
1314 int64_t valid_bits_offset,
1315 typename EncodingTraits<ByteArrayType>::Accumulator* out,
1316 int* out_num_values) {
1317 constexpr int32_t kBufferSize = 1024;
1318 int32_t indices[kBufferSize];
1319
1320 auto visit_binary_helper = [&](auto* helper) {
1321 const auto* dict_values = dictionary_->data_as<ByteArray>();
1322 const int values_to_decode = num_values - null_count;
1323 int values_decoded = 0;
1324 int num_indices = 0;
1325 int pos_indices = 0;
1326
1327 auto visit_bit_run = [&](int64_t position, int64_t length, bool valid) {
1328 if (valid) {
1329 while (length > 0) {
1330 if (num_indices == pos_indices) {
1331 // Refill indices buffer
1332 const auto max_batch_size =
1333 std::min<int32_t>(kBufferSize, values_to_decode - values_decoded);
1334 num_indices = idx_decoder_.GetBatch(indices, max_batch_size);
1335 if (ARROW_PREDICT_FALSE(num_indices < 1)) {
1336 return Status::Invalid("Invalid number of indices: ", num_indices);
1337 }
1338 pos_indices = 0;
1339 }
1340 const auto batch_size = std::min<int64_t>(num_indices - pos_indices, length);
1341 for (int64_t j = 0; j < batch_size; ++j) {
1342 const auto index = indices[pos_indices++];
1343 RETURN_NOT_OK(IndexInBounds(index));
1344 const auto& val = dict_values[index];
1345 RETURN_NOT_OK(helper->AppendValue(val.ptr, static_cast<int32_t>(val.len)));
1346 }
1347 values_decoded += static_cast<int32_t>(batch_size);
1348 length -= static_cast<int32_t>(batch_size);
1349 }
1350 } else {
1351 for (int64_t i = 0; i < length; ++i) {
1352 helper->UnsafeAppendNull();
1353 }
1354 }
1355 return Status::OK();
1356 };
1357
1358 RETURN_NOT_OK(
1359 VisitBitRuns(valid_bits, valid_bits_offset, num_values, visit_bit_run));
1360 *out_num_values = values_decoded;
1361 return Status::OK();
1362 };
1363 // The `len_` in the ByteArrayDictDecoder is the total length of the
1364 // RLE/Bit-pack encoded data size, so, we cannot use `len_` to reserve
1365 // space for binary data.
1366 return DispatchArrowBinaryHelper<ByteArrayType>(
1367 out, num_values, /*estimated_data_length=*/{}, visit_binary_helper);
1368 }
1369
1370 template <typename BuilderType>

Callers

nothing calls this directly

Calls 6

VisitBitRunsFunction · 0.85
InvalidFunction · 0.50
OKFunction · 0.50
GetBatchMethod · 0.45
AppendValueMethod · 0.45
UnsafeAppendNullMethod · 0.45

Tested by

no test coverage detected