Function GetUtf8CodePointCount

be/src/exprs/mask-functions-ir.cc:195–211

Counting code points in the UTF-8 encoded string using the same method, 'to_unicode', as MaskSubStrUtf8 uses. So we can have a consistent behavior. Returns -1 if the string contains malformed(illegal/incomplete) code points.

/// as MaskSubStrUtf8 uses. So we can have a consistent behavior.
/// Returns -1 if the string contains malformed(illegal/incomplete) code points.
static int GetUtf8CodePointCount(FunctionContext* ctx, const StringVal& val) {
  utf8_codecvt<char>::state_type cvt_state;
  const char* p = reinterpret_cast<char*>(val.ptr);
  const char* p_end = p + val.len;
  int char_cnt = 0;
  while (p != p_end) {
    uint32_t c = utf8_codecvt<char>::to_unicode(cvt_state, p, p_end);
    if (c == utf::illegal || c == utf::incomplete) {
      ctx->SetError(Substitute("The $0-th code point $1 is $2",
          char_cnt, AnyValUtil::ToString(val),
          c == utf::illegal ? "illegal" : "incomplete").c_str());
      return -1;
    }
    ++char_cnt;
  }
  return char_cnt;
}

/// Mask the given string except the first 'un_mask_char_count' chars. Ported from
/// org.apache.hadoop.hive.ql.udf.generic.GenericUDFMaskShowFirstN.