| 383 | |
| 384 | |
| 385 | std::u32string UTF8toUTF32(const std::string& s8) { |
| 386 | std::u32string s32; |
| 387 | // Pre-allocate maximum possible size |
| 388 | s32.reserve(s8.size()); |
| 389 | |
| 390 | for (size_t i = 0; i < s8.size();) { |
| 391 | char32_t c32; |
| 392 | size_t size; |
| 393 | |
| 394 | // Determine the number of bytes in the UTF-8 sequence |
| 395 | if ((s8[i] & 0x80) == 0x00) { |
| 396 | // 1-byte sequence |
| 397 | c32 = s8[i]; |
| 398 | size = 1; |
| 399 | } |
| 400 | else if ((s8[i] & 0xE0) == 0xC0) { |
| 401 | // 2-byte sequence |
| 402 | if (i + 1 >= s8.size()) |
| 403 | break; |
| 404 | c32 = (char32_t(s8[i] & 0x1F) << 6) | |
| 405 | (char32_t(s8[i + 1]) & 0x3F); |
| 406 | size = 2; |
| 407 | // Ignore overlong sequence |
| 408 | if (c32 < 0x80) |
| 409 | continue; |
| 410 | } |
| 411 | else if ((s8[i] & 0xF0) == 0xE0) { |
| 412 | // 3-byte sequence |
| 413 | if (i + 2 >= s8.size()) |
| 414 | break; |
| 415 | c32 = (char32_t(s8[i] & 0x0F) << 12) | |
| 416 | ((char32_t(s8[i + 1]) & 0x3F) << 6) | |
| 417 | (char32_t(s8[i + 2]) & 0x3F); |
| 418 | size = 3; |
| 419 | // Ignore overlong sequence |
| 420 | if (c32 < 0x800) |
| 421 | continue; |
| 422 | // Ignore surrogate pairs |
| 423 | if (c32 >= 0xD800 && c32 <= 0xDFFF) |
| 424 | continue; |
| 425 | } |
| 426 | else if ((s8[i] & 0xF8) == 0xF0) { |
| 427 | // 4-byte sequence |
| 428 | if (i + 3 >= s8.size()) |
| 429 | break; |
| 430 | c32 = (char32_t(s8[i] & 0x07) << 18) | |
| 431 | ((char32_t(s8[i + 1]) & 0x3F) << 12) | |
| 432 | ((char32_t(s8[i + 2]) & 0x3F) << 6) | |
| 433 | (char32_t(s8[i + 3]) & 0x3F); |
| 434 | size = 4; |
| 435 | |
| 436 | // Ignore overlong sequence |
| 437 | if (c32 < 0x10000) |
| 438 | continue; |
| 439 | // Ignore codepoints beyond Unicode maximum |
| 440 | if (c32 > 0x10FFFF) |
| 441 | continue; |
| 442 | } |