MCPcopy Create free account
hub / github.com/apache/arrow / utf8_length

Function utf8_length

cpp/src/gandiva/precompiled/string_ops.cc:177–196  ·  view source on GitHub ↗

Count the number of utf8 characters return 0 for invalid/incomplete input byte sequences

Source from the content-addressed store, hash-verified

175// Count the number of utf8 characters
176// return 0 for invalid/incomplete input byte sequences
177FORCE_INLINE
178gdv_int32 utf8_length(gdv_int64 context, const char* data, gdv_int32 data_len) {
179 int char_len = 0;
180 int count = 0;
181 for (int i = 0; i < data_len; i += char_len) {
182 char_len = utf8_char_length(data[i]);
183 if (char_len == 0 || i + char_len > data_len) { // invalid byte or incomplete glyph
184 set_error_for_invalid_utf(context, data[i]);
185 return 0;
186 }
187 for (int j = 1; j < char_len; ++j) {
188 if ((data[i + j] & 0xC0) != 0x80) { // bytes following head-byte of glyph
189 set_error_for_invalid_utf(context, data[i + j]);
190 return 0;
191 }
192 }
193 ++count;
194 }
195 return count;
196}
197
198// Count the number of utf8 characters, ignoring invalid char, considering size 1
199FORCE_INLINE

Callers 5

substr_utf8_int64_int64Function · 0.85
locate_utf8_utf8_int32Function · 0.85
left_utf8_int32Function · 0.85
right_utf8_int32Function · 0.85
TESTFunction · 0.85

Calls

no outgoing calls

Tested by 1

TESTFunction · 0.68