Convert UTF-8 to 32-bit character, process single character input. A nearly-branchless UTF-8 decoder, based on work of Christopher Wellons (https://github.com/skeeto/branchless-utf8). We handle UTF-8 decoding error by skipping forward.
| 1614 | // A nearly-branchless UTF-8 decoder, based on work of Christopher Wellons (https://github.com/skeeto/branchless-utf8). |
| 1615 | // We handle UTF-8 decoding error by skipping forward. |
| 1616 | int ImTextCharFromUtf8(unsigned int* out_char, const char* in_text, const char* in_text_end) |
| 1617 | { |
| 1618 | static const char lengths[32] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 3, 3, 4, 0 }; |
| 1619 | static const int masks[] = { 0x00, 0x7f, 0x1f, 0x0f, 0x07 }; |
| 1620 | static const uint32_t mins[] = { 0x400000, 0, 0x80, 0x800, 0x10000 }; |
| 1621 | static const int shiftc[] = { 0, 18, 12, 6, 0 }; |
| 1622 | static const int shifte[] = { 0, 6, 4, 2, 0 }; |
| 1623 | int len = lengths[*(const unsigned char*)in_text >> 3]; |
| 1624 | int wanted = len + !len; |
| 1625 | |
| 1626 | if (in_text_end == NULL) |
| 1627 | in_text_end = in_text + wanted; // Max length, nulls will be taken into account. |
| 1628 | |
| 1629 | // Copy at most 'len' bytes, stop copying at 0 or past in_text_end. Branch predictor does a good job here, |
| 1630 | // so it is fast even with excessive branching. |
| 1631 | unsigned char s[4]; |
| 1632 | s[0] = in_text + 0 < in_text_end ? in_text[0] : 0; |
| 1633 | s[1] = in_text + 1 < in_text_end ? in_text[1] : 0; |
| 1634 | s[2] = in_text + 2 < in_text_end ? in_text[2] : 0; |
| 1635 | s[3] = in_text + 3 < in_text_end ? in_text[3] : 0; |
| 1636 | |
| 1637 | // Assume a four-byte character and load four bytes. Unused bits are shifted out. |
| 1638 | *out_char = (uint32_t)(s[0] & masks[len]) << 18; |
| 1639 | *out_char |= (uint32_t)(s[1] & 0x3f) << 12; |
| 1640 | *out_char |= (uint32_t)(s[2] & 0x3f) << 6; |
| 1641 | *out_char |= (uint32_t)(s[3] & 0x3f) << 0; |
| 1642 | *out_char >>= shiftc[len]; |
| 1643 | |
| 1644 | // Accumulate the various error conditions. |
| 1645 | int e = 0; |
| 1646 | e = (*out_char < mins[len]) << 6; // non-canonical encoding |
| 1647 | e |= ((*out_char >> 11) == 0x1b) << 7; // surrogate half? |
| 1648 | e |= (*out_char > IM_UNICODE_CODEPOINT_MAX) << 8; // out of range? |
| 1649 | e |= (s[1] & 0xc0) >> 2; |
| 1650 | e |= (s[2] & 0xc0) >> 4; |
| 1651 | e |= (s[3] ) >> 6; |
| 1652 | e ^= 0x2a; // top two bits of each tail byte correct? |
| 1653 | e >>= shifte[len]; |
| 1654 | |
| 1655 | if (e) |
| 1656 | { |
| 1657 | // No bytes are consumed when *in_text == 0 || in_text == in_text_end. |
| 1658 | // One byte is consumed in case of invalid first byte of in_text. |
| 1659 | // All available bytes (at most `len` bytes) are consumed on incomplete/invalid second to last bytes. |
| 1660 | // Invalid or incomplete input may consume less bytes than wanted, therefore every byte has to be inspected in s. |
| 1661 | wanted = ImMin(wanted, !!s[0] + !!s[1] + !!s[2] + !!s[3]); |
| 1662 | *out_char = IM_UNICODE_CODEPOINT_INVALID; |
| 1663 | } |
| 1664 | |
| 1665 | return wanted; |
| 1666 | } |
| 1667 | |
| 1668 | int ImTextStrFromUtf8(ImWchar* buf, int buf_size, const char* in_text, const char* in_text_end, const char** in_text_remaining) |
| 1669 | { |
no test coverage detected