| 262 | } |
| 263 | |
| 264 | void bytes_to_bits(int64_t hardware_flags, const int num_bits, const uint8_t* bytes, |
| 265 | uint8_t* bits, int bit_offset) { |
| 266 | bits += bit_offset / 8; |
| 267 | bit_offset %= 8; |
| 268 | if (bit_offset != 0) { |
| 269 | uint64_t bits_head; |
| 270 | int bits_in_first_byte = std::min(num_bits, 8 - bit_offset); |
| 271 | bytes_to_bits(hardware_flags, bits_in_first_byte, bytes, |
| 272 | reinterpret_cast<uint8_t*>(&bits_head)); |
| 273 | uint8_t mask = (1 << bit_offset) - 1; |
| 274 | *bits = static_cast<uint8_t>((*bits & mask) | (bits_head << bit_offset)); |
| 275 | |
| 276 | if (num_bits > bits_in_first_byte) { |
| 277 | bytes_to_bits(hardware_flags, num_bits - bits_in_first_byte, |
| 278 | bytes + bits_in_first_byte, bits + 1); |
| 279 | } |
| 280 | return; |
| 281 | } |
| 282 | |
| 283 | int num_processed = 0; |
| 284 | #if defined(ARROW_HAVE_RUNTIME_AVX2) && defined(ARROW_HAVE_RUNTIME_BMI2) |
| 285 | if ((hardware_flags & CpuInfo::AVX2) && CpuInfo::GetInstance()->HasEfficientBmi2()) { |
| 286 | // The function call below processes whole 32 bit chunks together. |
| 287 | num_processed = num_bits - (num_bits % 32); |
| 288 | avx2::bytes_to_bits_avx2(num_processed, bytes, bits); |
| 289 | } |
| 290 | #endif |
| 291 | // Process 8 bits at a time |
| 292 | constexpr int unroll = 8; |
| 293 | for (int i = num_processed / unroll; i < num_bits / unroll; ++i) { |
| 294 | uint64_t bytes_next = util::SafeLoad(&reinterpret_cast<const uint64_t*>(bytes)[i]); |
| 295 | bytes_next &= 0x0101010101010101ULL; |
| 296 | bytes_next |= (bytes_next >> 7); // Pairs of adjacent output bits in individual bytes |
| 297 | bytes_next |= (bytes_next >> 14); // 4 adjacent output bits in individual bytes |
| 298 | bytes_next |= (bytes_next >> 28); // All 8 output bits in the lowest byte |
| 299 | bits[i] = static_cast<uint8_t>(bytes_next & 0xff); |
| 300 | } |
| 301 | int tail = num_bits % unroll; |
| 302 | if (tail) { |
| 303 | uint64_t bytes_next = SafeLoadUpTo8Bytes(bytes + num_bits - tail, tail); |
| 304 | bytes_next &= 0x0101010101010101ULL; |
| 305 | bytes_next |= (bytes_next >> 7); // Pairs of adjacent output bits in individual bytes |
| 306 | bytes_next |= (bytes_next >> 14); // 4 adjacent output bits in individual bytes |
| 307 | bytes_next |= (bytes_next >> 28); // All 8 output bits in the lowest byte |
| 308 | bits[num_bits / 8] = static_cast<uint8_t>(bytes_next & 0xff); |
| 309 | } |
| 310 | } |
| 311 | |
| 312 | bool are_all_bytes_zero(int64_t hardware_flags, const uint8_t* bytes, |
| 313 | uint32_t num_bytes) { |
no test coverage detected