Validates whether a given string is UTF-8 encoded.
| 121 | |
| 122 | /// Validates whether a given string is UTF-8 encoded. |
| 123 | bool validate_utf8(const char* str, bool restrict_to_two_bytes) noexcept { |
| 124 | if (!str) { |
| 125 | return false; |
| 126 | } |
| 127 | |
| 128 | constexpr uint8_t UTF8_2BYTE_MASK = 0xE0; |
| 129 | constexpr uint8_t UTF8_2BYTE_PREFIX = 0xC0; |
| 130 | constexpr uint8_t UTF8_3BYTE_MASK = 0xF0; |
| 131 | constexpr uint8_t UTF8_3BYTE_PREFIX = 0xE0; |
| 132 | constexpr uint8_t UTF8_4BYTE_MASK = 0xF8; |
| 133 | constexpr uint8_t UTF8_4BYTE_PREFIX = 0xF0; |
| 134 | constexpr uint8_t UTF8_CONTINUATION_MASK = 0xC0; |
| 135 | constexpr uint8_t UTF8_CONTINUATION_PREFIX = 0x80; |
| 136 | |
| 137 | // bool has_two_byte = false; |
| 138 | const auto* p = reinterpret_cast<const unsigned char*>(str); |
| 139 | |
| 140 | while (*p) { |
| 141 | uint8_t c = *p; |
| 142 | |
| 143 | // Single-byte ASCII (U+0000 - U+007F) |
| 144 | if (c <= 0x7F) { |
| 145 | ++p; |
| 146 | continue; |
| 147 | } |
| 148 | |
| 149 | // Detect standalone high bytes (0x80-0xFF), common in Windows-1252 |
| 150 | if (c >= 0x80 && (c < 0xC2 || (c & UTF8_2BYTE_MASK) != UTF8_2BYTE_PREFIX) && (c & UTF8_3BYTE_MASK) != UTF8_3BYTE_PREFIX && |
| 151 | (c & UTF8_4BYTE_MASK) != UTF8_4BYTE_PREFIX) { |
| 152 | return false; // Invalid UTF-8 start byte, likely ANSI |
| 153 | } |
| 154 | |
| 155 | // 2-byte sequence (U+0080 - U+07FF) |
| 156 | if ((c & UTF8_2BYTE_MASK) == UTF8_2BYTE_PREFIX) { |
| 157 | if (c < 0xC2 || !p[1]) { |
| 158 | return false; // Overlong encoding or missing continuation byte |
| 159 | } |
| 160 | uint8_t c1 = p[1]; |
| 161 | if ((c1 & UTF8_CONTINUATION_MASK) != UTF8_CONTINUATION_PREFIX) { |
| 162 | // Check for ANSI-like second byte (0x40-0x7F), common in GBK/Shift-JIS |
| 163 | if (c1 >= 0x40 && c1 <= 0x7F) { |
| 164 | return false; // Likely GBK or Shift-JIS, not UTF-8 |
| 165 | } |
| 166 | return false; // Invalid continuation byte |
| 167 | } |
| 168 | // Decode and check code point to exclude rare ranges |
| 169 | uint32_t code_point = ((c & 0x1F) << 6) | (c1 & 0x3F); |
| 170 | if (code_point < 0x80 || code_point > 0x7FF || (code_point >= 0x0080 && code_point <= 0x05FF)) { |
| 171 | return false; // Invalid or rare code point, likely ANSI |
| 172 | } |
| 173 | // has_two_byte = true; |
| 174 | p += 2; |
| 175 | continue; |
| 176 | } |
| 177 | |
| 178 | // Restrict to 2-byte sequences if specified |
| 179 | if (restrict_to_two_bytes) { |
| 180 | return false; |