Function MaskSubStrUtf8

be/src/exprs/mask-functions-ir.cc:126–190

Mask the substring in range [start, end) of the given string value. Using rules in 'MaskTransform'. Indices are counted in UTF-8 code points.

/// Mask the substring in range [start, end) of the given string value. Using rules in
/// 'MaskTransform'. Indices are counted in UTF-8 code points.
static StringVal MaskSubStrUtf8(FunctionContext* ctx, const StringVal& val,
    int start, int end, int masked_upper_char, int masked_lower_char,
    int masked_digit_char, int masked_other_char) {
  DCHECK_GE(start, 0);
  DCHECK_LT(start, end);
  DCHECK_LE(end, val.len);
  const char* p_start = reinterpret_cast<char*>(val.ptr);
  const char* p_end = p_start + val.len;
  const char* p = p_start;
  utf8_codecvt<char>::state_type cvt_state;
  int char_cnt = 0;
  // Skip leading 'start' code points. Leading bytes will be copied directly.
  while (char_cnt < start && p != p_end) {
    uint32_t codepoint = utf8_codecvt<char>::to_unicode(cvt_state, p, p_end);
    if (CheckAndWarnCodePoint(ctx, codepoint)) return StringVal::null();
    ++char_cnt;
  }
  // Calculating the result length in bytes.
  int result_bytes = p - p_start;
  int leading_bytes = result_bytes;
  // Collect code points at range [start, end - 1) and mask them.
  vector<uint32_t> masked_code_points;
  // Create unicode locale for checking upper/lower cases or digits.
  static const std::locale& loc = boost::locale::generator()("en_US.UTF-8");
  // Check facet existence to avoid predicates throws exception.
  if (!std::has_facet<std::ctype<wchar_t>>(loc)) {
    ctx->SetError("Cannot mask unicode strings since locale en_US.UTF-8 not found!");
    return StringVal();
  }
  while (char_cnt < end && p != p_end) {
    // Parse and get the first code point in string range [p, p_end).
    // 'to_unicode' will update the pointer 'p'.
    uint32_t codepoint = utf8_codecvt<char>::to_unicode(cvt_state, p, p_end);
    if (CheckAndWarnCodePoint(ctx, codepoint)) return StringVal::null();
    codepoint = MaskTransform(codepoint, masked_upper_char, masked_lower_char,
        masked_digit_char, masked_other_char, &loc);
    masked_code_points.push_back(codepoint);
    result_bytes += utf::utf_traits<char>::width(codepoint);
    ++char_cnt;
  }
  // Trailing bytes will be copied directly without masking.
  int tail_len = p_end - p;
  result_bytes += tail_len;

  StringVal result(ctx, result_bytes);
  if (UNLIKELY(result.is_null)) return result;
  // Copy leading bytes.
  Ubsan::MemCpy(result.ptr, val.ptr, leading_bytes);
  // Converting masked code points to UTF-8 encoded bytes.
  char* ptr = reinterpret_cast<char*>(result.ptr) + leading_bytes;
  p_end = reinterpret_cast<char*>(result.ptr) + result_bytes;
  for (uint32_t c : masked_code_points) {
    uint32_t width = utf8_codecvt<char>::from_unicode(cvt_state, c, ptr, p_end);
    DCHECK(width != utf::illegal && width != utf::incomplete);
    ptr += width;
    DCHECK(ptr <= p_end);
  }
  // Copy trailing bytes.