| 48 | |
| 49 | template <typename Arch, int kNumStreams> |
| 50 | void ByteStreamSplitDecodeSimd(const uint8_t* data, int width, int64_t num_values, |
| 51 | int64_t stride, uint8_t* out) { |
| 52 | using simd_batch = xsimd::batch<int8_t, Arch>; |
| 53 | // For signed arithmetic |
| 54 | constexpr int kBatchSize = static_cast<int>(simd_batch::size); |
| 55 | |
| 56 | static_assert(kBatchSize >= 16, "The smallest SIMD size is 128 bits"); |
| 57 | |
| 58 | if constexpr (kBatchSize > 16) { |
| 59 | if (num_values < kBatchSize) { |
| 60 | using Arch128 = xsimd::make_sized_batch_t<int8_t, 16>::arch_type; |
| 61 | return ByteStreamSplitDecodeSimd<Arch128, kNumStreams>(data, width, num_values, |
| 62 | stride, out); |
| 63 | } |
| 64 | } |
| 65 | |
| 66 | static_assert(kNumStreams <= kBatchSize, |
| 67 | "The algorithm works when the number of streams is smaller than the SIMD " |
| 68 | "batch size."); |
| 69 | assert(width == kNumStreams); |
| 70 | constexpr int kNumStreamsLog2 = ReversePow2(kNumStreams); |
| 71 | static_assert(kNumStreamsLog2 != 0, |
| 72 | "The algorithm works for a number of streams being a power of two."); |
| 73 | constexpr int64_t kBlockSize = kBatchSize * kNumStreams; |
| 74 | |
| 75 | const int64_t size = num_values * kNumStreams; |
| 76 | const int64_t num_blocks = size / kBlockSize; |
| 77 | |
| 78 | // First handle suffix. |
| 79 | // This helps catch if the simd-based processing overflows into the suffix |
| 80 | // since almost surely a test would fail. |
| 81 | const int64_t num_processed_elements = (num_blocks * kBlockSize) / kNumStreams; |
| 82 | for (int64_t i = num_processed_elements; i < num_values; ++i) { |
| 83 | uint8_t gathered_byte_data[kNumStreams]; |
| 84 | for (int b = 0; b < kNumStreams; ++b) { |
| 85 | const int64_t byte_index = b * stride + i; |
| 86 | gathered_byte_data[b] = data[byte_index]; |
| 87 | } |
| 88 | std::memcpy(out + i * kNumStreams, gathered_byte_data, kNumStreams); |
| 89 | } |
| 90 | |
| 91 | // The blocks get processed hierarchically using the unpack intrinsics. |
| 92 | // Example with four streams: |
| 93 | // Stage 1: AAAA BBBB CCCC DDDD |
| 94 | // Stage 2: ACAC ACAC BDBD BDBD |
| 95 | // Stage 3: ABCD ABCD ABCD ABCD |
| 96 | constexpr int kNumStreamsHalf = kNumStreams / 2U; |
| 97 | |
| 98 | for (int64_t block_index = 0; block_index < num_blocks; ++block_index) { |
| 99 | simd_batch stage[kNumStreamsLog2 + 1][kNumStreams]; |
| 100 | |
| 101 | for (int i = 0; i < kNumStreams; ++i) { |
| 102 | stage[0][i] = |
| 103 | simd_batch::load_unaligned(&data[block_index * kBatchSize + i * stride]); |
| 104 | } |
| 105 | |
| 106 | for (int step = 0; step < kNumStreamsLog2; ++step) { |
| 107 | for (int i = 0; i < kNumStreamsHalf; ++i) { |
nothing calls this directly
no test coverage detected