MCPcopy Create free account
hub / github.com/alibaba/MNN / parse_class

Method parse_class

transformers/llm/engine/src/tokenizer/unicode.cpp:293–357  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

291 }
292
293 RNode parse_class() {
294 next(); // skip '['
295 bool negated = (peek() == '^');
296 if (negated) next();
297 CharSet cs;
298 // Handle ']' as first char in class (literal)
299 if (peek() == ']') { cs.set((uint8_t)']'); next(); }
300 while (!at_end() && peek() != ']') {
301 if (peek() == '\\') {
302 add_escape_to_cs(cs);
303 } else if (is_multibyte()) {
304 // Multi-byte UTF-8 character in class
305 int32_t cp = read_cp();
306 uint8_t fb = cp_to_folded(cp);
307 if (peek() == '-' && pos_ + 1 < pat_.size() && pat_[pos_ + 1] != ']') {
308 next(); // skip '-'
309 int32_t end_cp;
310 if (is_multibyte()) {
311 end_cp = read_cp();
312 } else {
313 end_cp = (int32_t)(uint8_t)next();
314 }
315 uint8_t fb_end = cp_to_folded(end_cp);
316 // Set all folded bytes in the range
317 for (int b = (fb < fb_end ? fb : fb_end); b <= (fb < fb_end ? fb_end : fb); b++) {
318 cs.set((uint8_t)b);
319 }
320 } else {
321 cs.set(fb);
322 }
323 } else {
324 char c = next();
325 if (peek() == '-' && pos_ + 1 < pat_.size() && pat_[pos_ + 1] != ']') {
326 next(); // skip '-'
327 char end_c;
328 if (peek() == '\\') {
329 // Escaped range end
330 next(); end_c = next();
331 if (end_c == 'n') end_c = '\n';
332 else if (end_c == 'r') end_c = '\r';
333 else if (end_c == 't') end_c = '\t';
334 } else if (is_multibyte()) {
335 int32_t end_cp = read_cp();
336 uint8_t fb = cp_to_folded((int32_t)(uint8_t)c);
337 uint8_t fb_end = cp_to_folded(end_cp);
338 for (int b = (fb < fb_end ? fb : fb_end); b <= (fb < fb_end ? fb_end : fb); b++) {
339 cs.set((uint8_t)b);
340 }
341 continue;
342 } else {
343 end_c = next();
344 }
345 for (int b = (uint8_t)c; b <= (uint8_t)end_c; b++) {
346 add_literal(cs, (char)b);
347 }
348 } else {
349 add_literal(cs, c);
350 }

Callers

nothing calls this directly

Calls 5

cp_to_foldedFunction · 0.85
nextFunction · 0.50
setMethod · 0.45
sizeMethod · 0.45
negateMethod · 0.45

Tested by

no test coverage detected