| 84 | |
| 85 | template <int bit_to_search, bool filter_input_indexes> |
| 86 | void bits_to_indexes_internal(int64_t hardware_flags, const int num_bits, |
| 87 | const uint8_t* bits, const uint16_t* input_indexes, |
| 88 | int* num_indexes, uint16_t* indexes, |
| 89 | uint16_t base_index = 0) { |
| 90 | // 64 bits at a time |
| 91 | constexpr int unroll = 64; |
| 92 | int tail = num_bits % unroll; |
| 93 | #if defined(ARROW_HAVE_RUNTIME_AVX2) && defined(ARROW_HAVE_RUNTIME_BMI2) |
| 94 | if ((hardware_flags & CpuInfo::AVX2) && CpuInfo::GetInstance()->HasEfficientBmi2()) { |
| 95 | if (filter_input_indexes) { |
| 96 | avx2::bits_filter_indexes_avx2(bit_to_search, num_bits - tail, bits, input_indexes, |
| 97 | num_indexes, indexes); |
| 98 | } else { |
| 99 | avx2::bits_to_indexes_avx2(bit_to_search, num_bits - tail, bits, num_indexes, |
| 100 | indexes, base_index); |
| 101 | } |
| 102 | } else { |
| 103 | #endif |
| 104 | *num_indexes = 0; |
| 105 | for (int i = 0; i < num_bits / unroll; ++i) { |
| 106 | uint64_t word = util::SafeLoad(&reinterpret_cast<const uint64_t*>(bits)[i]); |
| 107 | if (bit_to_search == 0) { |
| 108 | word = ~word; |
| 109 | } |
| 110 | if (filter_input_indexes) { |
| 111 | bits_filter_indexes_helper(word, input_indexes + i * 64, num_indexes, indexes); |
| 112 | } else { |
| 113 | bits_to_indexes_helper(word, i * 64 + base_index, num_indexes, indexes); |
| 114 | } |
| 115 | } |
| 116 | #if defined(ARROW_HAVE_RUNTIME_AVX2) && defined(ARROW_HAVE_RUNTIME_BMI2) |
| 117 | } |
| 118 | #endif |
| 119 | // Optionally process the last partial word with masking out bits outside range |
| 120 | if (tail) { |
| 121 | const uint8_t* bits_tail = bits + (num_bits - tail) / 8; |
| 122 | uint64_t word = SafeLoadUpTo8Bytes(bits_tail, (tail + 7) / 8); |
| 123 | if (bit_to_search == 0) { |
| 124 | word = ~word; |
| 125 | } |
| 126 | word &= ~0ULL >> (64 - tail); |
| 127 | if (filter_input_indexes) { |
| 128 | bits_filter_indexes_helper(word, input_indexes + num_bits - tail, num_indexes, |
| 129 | indexes); |
| 130 | } else { |
| 131 | bits_to_indexes_helper(word, num_bits - tail + base_index, num_indexes, indexes); |
| 132 | } |
| 133 | } |
| 134 | } |
| 135 | |
| 136 | void bits_to_indexes(int bit_to_search, int64_t hardware_flags, int num_bits, |
| 137 | const uint8_t* bits, int* num_indexes, uint16_t* indexes, |
| 138 | int bit_offset) { |
| 139 | bits += bit_offset / 8; |
| 140 | bit_offset %= 8; |
| 141 | *num_indexes = 0; |
| 142 | uint16_t base_index = 0; |
| 143 | if (bit_offset != 0) { |
nothing calls this directly
no test coverage detected