MCPcopy Create free account
hub / github.com/colbymchenry/codegraph / scan_token

Function scan_token

codegraph-kernel/grammars/kotlin/scanner.c:1474–1632  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1472}
1473
1474static bool scan_token(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
1475 // Pending work takes priority over constructor, ASI, string and
1476 // interpolation paths. No other branch may consume its source as padding.
1477 if (scanner->comment.pending)
1478 return resume_pending_comment(scanner, lexer, valid_symbols);
1479 if (valid_symbols[DELEGATE_START_HINT] && !valid_symbols[STRING_CONTENT]) {
1480 lexer->mark_end(lexer);
1481 scanner->in_delegate = true;
1482 lexer->result_symbol = DELEGATE_START_HINT;
1483 return true;
1484 }
1485 if (valid_symbols[DELEGATE_END_HINT] && !valid_symbols[STRING_CONTENT]) {
1486 lexer->mark_end(lexer);
1487 scanner->in_delegate = false;
1488 lexer->result_symbol = DELEGATE_END_HINT;
1489 return true;
1490 }
1491 if (valid_symbols[PRIMARY_CONSTRUCTOR_HINT] && !valid_symbols[STRING_CONTENT])
1492 return scan_primary_constructor_hint(scanner, lexer, valid_symbols);
1493 scanner->constructor_newline = false;
1494 // TRY_CLAUSE_HINT is optional before catch/finally and is never emitted.
1495 // Its validity identifies clause positions without changing keyword nodes.
1496 // BY_DELEGATION_HINT is declared in the grammar (optional, before `by` in
1497 // explicit_delegation and property_delegate) purely so it appears in
1498 // valid_symbols when the parser is in a delegation context. The scanner
1499 // never emits it; it's used only as a context flag in scan_automatic_semicolon.
1500 if (valid_symbols[AUTOMATIC_SEMICOLON]) {
1501 bool cursor_clean = false;
1502 bool ret = scan_automatic_semicolon(scanner, lexer, valid_symbols, &cursor_clean);
1503 // If we fail to find an automatic semicolon, it's still possible that we may
1504 // want to lex a string or comment later -- but only if the ASI scan left the
1505 // cursor at the first non-whitespace character. Once it has skipped over
1506 // comment or keyword text, token_start has moved and any token produced
1507 // below would silently swallow everything in between as padding.
1508 //
1509 // Error recovery is exempt. There tree-sitter marks every external symbol
1510 // valid at once (STRING_CONTENT alongside AUTOMATIC_SEMICOLON is the
1511 // giveaway -- a statement can never end inside a string), and that is the
1512 // only way MULTILINE_COMMENT or STRING_START is reachable after a keyword
1513 // probe. The text is already bound for an ERROR node, so an external token
1514 // with a dragged start beats no token at all: without this exemption the
1515 // internal lexer tears an unterminated `/*` into `/` plus a bogus
1516 // wildcard_import, and a nested comment collapses to zero width.
1517 if (ret) return ret;
1518 if (!cursor_clean && !valid_symbols[STRING_CONTENT]) return false;
1519 }
1520
1521 // A backtick-quoted identifier. The internal lexer cannot express this:
1522 // a character class stops at the NUL that terminates its input buffer, so
1523 // `a<NUL>b` tore into pieces, while Kotlin's lexer accepts any character
1524 // here except a backtick or a line terminator. Scanning it externally and
1525 // testing `eof` rather than a sentinel byte carries the NUL through.
1526 if (valid_symbols[QUOTED_IDENTIFIER] && !valid_symbols[STRING_CONTENT]) {
1527 while (iswspace(lexer->lookahead)) skip(lexer);
1528 if (lexer->lookahead == '`') {
1529 advance(lexer);
1530 bool content = false;
1531 while (!lexer->eof(lexer) && lexer->lookahead != '`' &&

Calls 10

resume_pending_commentFunction · 0.85
scan_automatic_semicolonFunction · 0.85
scan_import_dotFunction · 0.85
constructor_word_charFunction · 0.85
scan_string_startFunction · 0.85
scan_multiline_commentFunction · 0.85
skipFunction · 0.70
advanceFunction · 0.70
scan_string_contentFunction · 0.70

Tested by

no test coverage detected