Reads the next token, updating the Lexer's token fields. The end state is EOF, after which any further calls to nextToken() will produce only EOF.
()
| 121 | * further calls to {@code nextToken()} will produce only EOF. |
| 122 | */ |
| 123 | void nextToken() { |
| 124 | boolean afterNewline = kind == TokenKind.NEWLINE || kind == TokenKind.DOC_COMMENT_BLOCK; |
| 125 | tokenize(); |
| 126 | Preconditions.checkState(kind != null); |
| 127 | |
| 128 | // Always end with a NEWLINE (or DOC_COMMENT_BLOCK) token, even if no '\n' in input, to simplify |
| 129 | // parser's logic. (Note that Python also always ends with a NEWLINE.) |
| 130 | if (kind == TokenKind.EOF && !afterNewline) { |
| 131 | kind = TokenKind.NEWLINE; |
| 132 | } |
| 133 | if (kind != TokenKind.NEWLINE |
| 134 | && kind != TokenKind.INDENT |
| 135 | && kind != TokenKind.OUTDENT |
| 136 | && kind != TokenKind.DOC_COMMENT_BLOCK |
| 137 | && kind != TokenKind.DOC_COMMENT_TRAILING) { |
| 138 | lineOnlyWhitespaceOrComments = false; |
| 139 | } |
| 140 | } |
| 141 | |
| 142 | private void popParen() { |
| 143 | if (openParenStackDepth == 0) { |