Function utf8_to_utf16_simd

cpp/fory/util/string_util.cc:28–150

namespace fory {

std::u16string utf8_to_utf16_simd(const std::string &utf8,
                                  bool is_little_endian) {
  std::u16string utf16;
  utf16.reserve(utf8.size()); // reserve space to avoid frequent reallocations

  char buffer[64]; // Buffer to hold temporary UTF-16 results
  char16_t *output =
      reinterpret_cast<char16_t *>(buffer); // Use char16_t for output

  size_t i = 0;
  size_t n = utf8.size();

  while (i + 32 <= n) {

    for (int j = 0; j < 32; ++j) {
      uint8_t byte = utf8[i + j];

      if (byte < 0x80) {
        // 1-byte character (ASCII)
        *output++ = static_cast<char16_t>(byte);
      } else if (byte < 0xE0) {
        // 2-byte character
        uint16_t utf16_char = ((byte & 0x1F) << 6) | (utf8[i + j + 1] & 0x3F);
        if (!is_little_endian) {
          utf16_char = (utf16_char >> 8) |
                       (utf16_char << 8); // Swap bytes for big-endian
        }
        *output++ = utf16_char;
        ++j;
      } else if (byte < 0xF0) {
        // 3-byte character
        uint16_t utf16_char = ((byte & 0x0F) << 12) |
                              ((utf8[i + j + 1] & 0x3F) << 6) |
                              (utf8[i + j + 2] & 0x3F);
        if (!is_little_endian) {
          utf16_char = (utf16_char >> 8) |
                       (utf16_char << 8); // Swap bytes for big-endian
        }
        *output++ = utf16_char;
        j += 2;
      } else {
        // 4-byte character (surrogate pair handling required)
        uint32_t code_point =
            ((byte & 0x07) << 18) | ((utf8[i + j + 1] & 0x3F) << 12) |
            ((utf8[i + j + 2] & 0x3F) << 6) | (utf8[i + j + 3] & 0x3F);

        // Convert the code point to a surrogate pair
        uint16_t high_surrogate = 0xD800 + ((code_point - 0x10000) >> 10);
        uint16_t low_surrogate = 0xDC00 + (code_point & 0x3FF);

        if (!is_little_endian) {
          high_surrogate = (high_surrogate >> 8) |
                           (high_surrogate << 8); // Swap bytes for big-endian
          low_surrogate = (low_surrogate >> 8) |
                          (low_surrogate << 8); // Swap bytes for big-endian
        }

        *output++ = high_surrogate;