Function Utf8CaseConversion

be/src/exprs/string-functions-ir.cc:382–441

Converts string based on the transform function 'fn'. The unit of the conversion is a wchar_t (i.e. uint32_t) which is parsed from multi bytes using std::mbtowc(). The transform function 'fn' accepts two parameters: the original wchar_t and a flag indicating whether it's the first character of a word. After the transformation, the wchar_t is converted back to bytes.

/// indicating whether it's the first character of a word.
/// After the transformation, the wchar_t is converted back to bytes.
static StringVal Utf8CaseConversion(FunctionContext* context, const StringVal& str,
    uint32_t (*fn)(uint32_t, bool*)) {
  // Usually the upper/lower cases have the same size in bytes. Here we add 4 bytes
  // buffer in case of illegal Unicodes.
  int max_result_bytes = str.len + 4;
  StringVal result(context, max_result_bytes);
  if (UNLIKELY(result.is_null)) return StringVal::null();
  wchar_t wc;
  int wc_bytes;
  bool word_start = true;
  uint8_t* result_ptr = result.ptr;
  std::mbstate_t wc_state{};
  std::mbstate_t mb_state{};
  for (int i = 0; i < str.len; i += wc_bytes) {
    // std::mbtowc converts a multibyte sequence to a wide character. It's not
    // thread safe. Here we use std::mbrtowc instead.
    wc_bytes = std::mbrtowc(&wc, reinterpret_cast<char*>(str.ptr + i), str.len - i,
        &wc_state);
    bool needs_conversion = true;
    if (wc_bytes == 0) {
      // std::mbtowc returns 0 when hitting '\0'.
      wc = 0;
      wc_bytes = 1;
    } else if (wc_bytes < 0) {
      ReportErrorBytes(context, str, i);
      // Replace it to the replacement character (U+FFFD)
      wc = 0xFFFD;
      needs_conversion = false;
      // Jump to the next legal UTF-8 start byte.
      wc_bytes = 1;
      while (i + wc_bytes < str.len && !BitUtil::IsUtf8StartByte(str.ptr[i + wc_bytes])) {
        wc_bytes++;
      }
    }
    if (needs_conversion) wc = fn(wc, &word_start);
    // std::wctomb converts a wide character to a multibyte sequence. It's not
    // thread safe. Here we use std::wcrtomb instead.
    int res_bytes = std::wcrtomb(reinterpret_cast<char*>(result_ptr), wc, &mb_state);
    if (res_bytes <= 0) {
      if (needs_conversion) {
        context->AddWarning(Substitute(
            "Ignored illegal wide character in results: $0. Current locale: $1",
            wc, std::locale("").name()).c_str());
      }
      continue;
    }
    result_ptr += res_bytes;
    if (result_ptr - result.ptr > max_result_bytes - 4) {
      // Double the result buffer for overflow
      max_result_bytes *= 2;
      max_result_bytes = min<int>(StringVal::MAX_LENGTH,
          static_cast<int>(BitUtil::RoundUpToPowerOfTwo(max_result_bytes)));
      int offset = result_ptr - result.ptr;
      if (UNLIKELY(!result.Resize(context, max_result_bytes))) return StringVal::null();
      result_ptr = result.ptr + offset;
    }
  }
  result.len = result_ptr - result.ptr;