MCPcopy Create free account
hub / github.com/alibaba/MNN / parse_atom

Method parse_atom

transformers/llm/engine/src/tokenizer/unicode.cpp:402–427  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

400 }
401
402 RNode parse_atom() {
403 char c = peek();
404 if (c == '(') return parse_group();
405 if (c == '[') return parse_class();
406 if (c == '\\') {
407 RNode n; n.type = RNode::CS;
408 add_escape_to_cs(n.cs);
409 return n;
410 }
411 if (c == '.') {
412 next();
413 RNode n; n.type = RNode::CS; n.cs = cs_dot();
414 return n;
415 }
416 // Literal character (handle multi-byte UTF-8)
417 if (is_multibyte()) {
418 int32_t cp = read_cp();
419 RNode n; n.type = RNode::CS;
420 n.cs.set(cp_to_folded(cp));
421 return n;
422 }
423 next();
424 RNode n; n.type = RNode::CS;
425 add_literal(n.cs, c);
426 return n;
427 }
428
429 void parse_quantifier(RNode& node) {
430 if (at_end()) return;

Callers

nothing calls this directly

Calls 4

cs_dotFunction · 0.85
cp_to_foldedFunction · 0.85
nextFunction · 0.50
setMethod · 0.45

Tested by

no test coverage detected