| 84 | } // namespace internal |
| 85 | |
| 86 | static inline bool ValidateUTF8Inline(const uint8_t* data, int64_t size) { |
| 87 | static constexpr uint64_t high_bits_64 = 0x8080808080808080ULL; |
| 88 | static constexpr uint32_t high_bits_32 = 0x80808080UL; |
| 89 | static constexpr uint16_t high_bits_16 = 0x8080U; |
| 90 | static constexpr uint8_t high_bits_8 = 0x80U; |
| 91 | |
| 92 | #ifndef NDEBUG |
| 93 | internal::CheckUTF8Initialized(); |
| 94 | #endif |
| 95 | |
| 96 | while (size >= 8) { |
| 97 | // XXX This is doing an unaligned access. Contemporary architectures |
| 98 | // (x86-64, AArch64, PPC64) support it natively and often have good |
| 99 | // performance nevertheless. |
| 100 | uint64_t mask64 = SafeLoadAs<uint64_t>(data); |
| 101 | if (ARROW_PREDICT_TRUE((mask64 & high_bits_64) == 0)) { |
| 102 | // 8 bytes of pure ASCII, move forward |
| 103 | size -= 8; |
| 104 | data += 8; |
| 105 | continue; |
| 106 | } |
| 107 | // Non-ASCII run detected. |
| 108 | // We process at least 4 bytes, to avoid too many spurious 64-bit reads |
| 109 | // in case the non-ASCII bytes are at the end of the tested 64-bit word. |
| 110 | // We also only check for rejection at the end since that state is stable |
| 111 | // (once in reject state, we always remain in reject state). |
| 112 | // It is guaranteed that size >= 8 when arriving here, which allows |
| 113 | // us to avoid size checks. |
| 114 | uint16_t state = internal::kUTF8ValidateAccept; |
| 115 | // Byte 0 |
| 116 | state = internal::ValidateOneUTF8Byte(*data++, state); |
| 117 | --size; |
| 118 | // Byte 1 |
| 119 | state = internal::ValidateOneUTF8Byte(*data++, state); |
| 120 | --size; |
| 121 | // Byte 2 |
| 122 | state = internal::ValidateOneUTF8Byte(*data++, state); |
| 123 | --size; |
| 124 | // Byte 3 |
| 125 | state = internal::ValidateOneUTF8Byte(*data++, state); |
| 126 | --size; |
| 127 | // Byte 4 |
| 128 | state = internal::ValidateOneUTF8Byte(*data++, state); |
| 129 | --size; |
| 130 | if (state == internal::kUTF8ValidateAccept) { |
| 131 | continue; // Got full char, switch back to ASCII detection |
| 132 | } |
| 133 | // Byte 5 |
| 134 | state = internal::ValidateOneUTF8Byte(*data++, state); |
| 135 | --size; |
| 136 | if (state == internal::kUTF8ValidateAccept) { |
| 137 | continue; // Got full char, switch back to ASCII detection |
| 138 | } |
| 139 | // Byte 6 |
| 140 | state = internal::ValidateOneUTF8Byte(*data++, state); |
| 141 | --size; |
| 142 | if (state == internal::kUTF8ValidateAccept) { |
| 143 | continue; // Got full char, switch back to ASCII detection |
no test coverage detected