MCPcopy Create free account
hub / github.com/apache/arrow / PutInternal

Method PutInternal

cpp/src/parquet/encoder.cc:1488–1531  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1486 protected:
1487 template <typename VisitorType>
1488 void PutInternal(const T* src, int num_values, const VisitorType visitor) {
1489 if (num_values == 0) {
1490 return;
1491 }
1492
1493 std::string_view last_value_view = last_value_;
1494 constexpr int kBatchSize = 256;
1495 std::array<int32_t, kBatchSize> prefix_lengths;
1496 std::array<ByteArray, kBatchSize> suffixes;
1497
1498 for (int i = 0; i < num_values; i += kBatchSize) {
1499 const int batch_size = std::min(kBatchSize, num_values - i);
1500
1501 for (int j = 0; j < batch_size; ++j) {
1502 const int idx = i + j;
1503 const auto view = visitor[idx];
1504 const auto len = static_cast<const uint32_t>(view.length());
1505
1506 uint32_t common_prefix_length = 0;
1507 const uint32_t maximum_common_prefix_length =
1508 std::min(len, static_cast<uint32_t>(last_value_view.length()));
1509 while (common_prefix_length < maximum_common_prefix_length) {
1510 if (last_value_view[common_prefix_length] != view[common_prefix_length]) {
1511 break;
1512 }
1513 common_prefix_length++;
1514 }
1515
1516 last_value_view = view;
1517 prefix_lengths[j] = common_prefix_length;
1518 const uint32_t suffix_length = len - common_prefix_length;
1519 const uint8_t* suffix_ptr = src[idx].ptr + common_prefix_length;
1520
1521 // Convert to ByteArray, so it can be passed to the suffix_encoder_.
1522 const ByteArray suffix(suffix_length, suffix_ptr);
1523 suffixes[j] = suffix;
1524
1525 unencoded_byte_array_data_bytes_ += len;
1526 }
1527 suffix_encoder_.Put(suffixes.data(), batch_size);
1528 prefix_length_encoder_.Put(prefix_lengths.data(), batch_size);
1529 }
1530 last_value_ = last_value_view;
1531 }
1532
1533 template <typename ArrayType>
1534 void PutBinaryArray(const ArrayType& array) {

Callers

nothing calls this directly

Calls 3

lengthMethod · 0.45
PutMethod · 0.45
dataMethod · 0.45

Tested by

no test coverage detected