| 1472 | } |
| 1473 | |
| 1474 | static bool scan_token(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) { |
| 1475 | // Pending work takes priority over constructor, ASI, string and |
| 1476 | // interpolation paths. No other branch may consume its source as padding. |
| 1477 | if (scanner->comment.pending) |
| 1478 | return resume_pending_comment(scanner, lexer, valid_symbols); |
| 1479 | if (valid_symbols[DELEGATE_START_HINT] && !valid_symbols[STRING_CONTENT]) { |
| 1480 | lexer->mark_end(lexer); |
| 1481 | scanner->in_delegate = true; |
| 1482 | lexer->result_symbol = DELEGATE_START_HINT; |
| 1483 | return true; |
| 1484 | } |
| 1485 | if (valid_symbols[DELEGATE_END_HINT] && !valid_symbols[STRING_CONTENT]) { |
| 1486 | lexer->mark_end(lexer); |
| 1487 | scanner->in_delegate = false; |
| 1488 | lexer->result_symbol = DELEGATE_END_HINT; |
| 1489 | return true; |
| 1490 | } |
| 1491 | if (valid_symbols[PRIMARY_CONSTRUCTOR_HINT] && !valid_symbols[STRING_CONTENT]) |
| 1492 | return scan_primary_constructor_hint(scanner, lexer, valid_symbols); |
| 1493 | scanner->constructor_newline = false; |
| 1494 | // TRY_CLAUSE_HINT is optional before catch/finally and is never emitted. |
| 1495 | // Its validity identifies clause positions without changing keyword nodes. |
| 1496 | // BY_DELEGATION_HINT is declared in the grammar (optional, before `by` in |
| 1497 | // explicit_delegation and property_delegate) purely so it appears in |
| 1498 | // valid_symbols when the parser is in a delegation context. The scanner |
| 1499 | // never emits it; it's used only as a context flag in scan_automatic_semicolon. |
| 1500 | if (valid_symbols[AUTOMATIC_SEMICOLON]) { |
| 1501 | bool cursor_clean = false; |
| 1502 | bool ret = scan_automatic_semicolon(scanner, lexer, valid_symbols, &cursor_clean); |
| 1503 | // If we fail to find an automatic semicolon, it's still possible that we may |
| 1504 | // want to lex a string or comment later -- but only if the ASI scan left the |
| 1505 | // cursor at the first non-whitespace character. Once it has skipped over |
| 1506 | // comment or keyword text, token_start has moved and any token produced |
| 1507 | // below would silently swallow everything in between as padding. |
| 1508 | // |
| 1509 | // Error recovery is exempt. There tree-sitter marks every external symbol |
| 1510 | // valid at once (STRING_CONTENT alongside AUTOMATIC_SEMICOLON is the |
| 1511 | // giveaway -- a statement can never end inside a string), and that is the |
| 1512 | // only way MULTILINE_COMMENT or STRING_START is reachable after a keyword |
| 1513 | // probe. The text is already bound for an ERROR node, so an external token |
| 1514 | // with a dragged start beats no token at all: without this exemption the |
| 1515 | // internal lexer tears an unterminated `/*` into `/` plus a bogus |
| 1516 | // wildcard_import, and a nested comment collapses to zero width. |
| 1517 | if (ret) return ret; |
| 1518 | if (!cursor_clean && !valid_symbols[STRING_CONTENT]) return false; |
| 1519 | } |
| 1520 | |
| 1521 | // A backtick-quoted identifier. The internal lexer cannot express this: |
| 1522 | // a character class stops at the NUL that terminates its input buffer, so |
| 1523 | // `a<NUL>b` tore into pieces, while Kotlin's lexer accepts any character |
| 1524 | // here except a backtick or a line terminator. Scanning it externally and |
| 1525 | // testing `eof` rather than a sentinel byte carries the NUL through. |
| 1526 | if (valid_symbols[QUOTED_IDENTIFIER] && !valid_symbols[STRING_CONTENT]) { |
| 1527 | while (iswspace(lexer->lookahead)) skip(lexer); |
| 1528 | if (lexer->lookahead == '`') { |
| 1529 | advance(lexer); |
| 1530 | bool content = false; |
| 1531 | while (!lexer->eof(lexer) && lexer->lookahead != '`' && |
no test coverage detected