MCPcopy Create free account
hub / github.com/apache/arrow / ByteStreamSplitDecodeSimd

Function ByteStreamSplitDecodeSimd

cpp/src/arrow/util/byte_stream_split_internal.h:50–121  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

48
49template <typename Arch, int kNumStreams>
50void ByteStreamSplitDecodeSimd(const uint8_t* data, int width, int64_t num_values,
51 int64_t stride, uint8_t* out) {
52 using simd_batch = xsimd::batch<int8_t, Arch>;
53 // For signed arithmetic
54 constexpr int kBatchSize = static_cast<int>(simd_batch::size);
55
56 static_assert(kBatchSize >= 16, "The smallest SIMD size is 128 bits");
57
58 if constexpr (kBatchSize > 16) {
59 if (num_values < kBatchSize) {
60 using Arch128 = xsimd::make_sized_batch_t<int8_t, 16>::arch_type;
61 return ByteStreamSplitDecodeSimd<Arch128, kNumStreams>(data, width, num_values,
62 stride, out);
63 }
64 }
65
66 static_assert(kNumStreams <= kBatchSize,
67 "The algorithm works when the number of streams is smaller than the SIMD "
68 "batch size.");
69 assert(width == kNumStreams);
70 constexpr int kNumStreamsLog2 = ReversePow2(kNumStreams);
71 static_assert(kNumStreamsLog2 != 0,
72 "The algorithm works for a number of streams being a power of two.");
73 constexpr int64_t kBlockSize = kBatchSize * kNumStreams;
74
75 const int64_t size = num_values * kNumStreams;
76 const int64_t num_blocks = size / kBlockSize;
77
78 // First handle suffix.
79 // This helps catch if the simd-based processing overflows into the suffix
80 // since almost surely a test would fail.
81 const int64_t num_processed_elements = (num_blocks * kBlockSize) / kNumStreams;
82 for (int64_t i = num_processed_elements; i < num_values; ++i) {
83 uint8_t gathered_byte_data[kNumStreams];
84 for (int b = 0; b < kNumStreams; ++b) {
85 const int64_t byte_index = b * stride + i;
86 gathered_byte_data[b] = data[byte_index];
87 }
88 std::memcpy(out + i * kNumStreams, gathered_byte_data, kNumStreams);
89 }
90
91 // The blocks get processed hierarchically using the unpack intrinsics.
92 // Example with four streams:
93 // Stage 1: AAAA BBBB CCCC DDDD
94 // Stage 2: ACAC ACAC BDBD BDBD
95 // Stage 3: ABCD ABCD ABCD ABCD
96 constexpr int kNumStreamsHalf = kNumStreams / 2U;
97
98 for (int64_t block_index = 0; block_index < num_blocks; ++block_index) {
99 simd_batch stage[kNumStreamsLog2 + 1][kNumStreams];
100
101 for (int i = 0; i < kNumStreams; ++i) {
102 stage[0][i] =
103 simd_batch::load_unaligned(&data[block_index * kBatchSize + i * stride]);
104 }
105
106 for (int step = 0; step < kNumStreamsLog2; ++step) {
107 for (int i = 0; i < kNumStreamsHalf; ++i) {

Callers

nothing calls this directly

Calls 1

ReversePow2Function · 0.85

Tested by

no test coverage detected