Function utf8string2WideString

mllm/preprocessor/tokenizers/Unicode.cpp:33–76

}

std::wstring utf8string2WideString(const std::string& str) {
  std::wstring w_ret_string;
  for (unsigned int i = 0; i < str.size();) {
    auto byte = static_cast<unsigned char>(str[i]);
    if ((byte & 0x80U) == 0) {
      // 1-byte character
      w_ret_string.push_back(static_cast<wchar_t>(byte));
      ++i;
    } else if ((byte & 0xE0U) == 0xC0) {
      // 2-byte character
      if (i + 1 < str.size()) {
        wchar_t wc = (static_cast<wchar_t>(byte & 0x1FU) << 6U) | (static_cast<wchar_t>(str[i + 1] & 0x3FU));
        w_ret_string.push_back(wc);
        i += 2;
      } else {
        break;
      }
    } else if ((byte & 0xF0U) == 0xE0U) {
      // 3-byte character
      if (i + 2 < str.size()) {
        wchar_t wc = (static_cast<wchar_t>(byte & 0x0FU) << 12U) | (static_cast<wchar_t>(str[i + 1] & 0x3FU) << 6U)
                     | (static_cast<wchar_t>(str[i + 2] & 0x3FU));
        w_ret_string.push_back(wc);
        i += 3;
      } else {
        break;
      }
    } else if ((byte & 0xF8U) == 0xF0U) {
      // 4-byte character
      if (i + 3 < str.size()) {
        wchar_t wc = (static_cast<wchar_t>(byte & 0x07U) << 18U) | (static_cast<wchar_t>(str[i + 1] & 0x3FU) << 12U)
                     | (static_cast<wchar_t>(str[i + 2] & 0x3FU) << 6U) | (static_cast<wchar_t>(str[i + 3] & 0x3FU));
        w_ret_string.push_back(wc);
        i += 4;
      } else {
        break;
      }
    } else {
      // Invalid UTF-8 sequence
      ++i;
    }
  }
  return w_ret_string;
}

void makeBytes2UnicodeMap(std::unordered_map<std::wint_t, wchar_t>& dict) {
  std::vector<std::wint_t> bs((ord(L"~") - ord(L"!") + 1) + (ord(L"¬") - ord(L"¡") + 1) + (ord(L"ÿ") - ord(L"®") + 1));