Method tokenize

src/main/java/net/starlark/java/syntax/Lexer.java:696–895

Scans for one token starting at the current position in the character buffer of file contents provided to the constructor. Updates the current token, and sets #pos to the next scanning position.

()

   * scanning position.
   */
  private void tokenize() {
    if (checkIndentation) {
      checkIndentation = false;
      computeIndentation();
    }

    // Return saved indentation tokens.
    if (dents != 0) {
      if (dents < 0) {
        dents++;
        setToken(TokenKind.OUTDENT, pos - 1, pos);
      } else {
        dents--;
        setToken(TokenKind.INDENT, pos - 1, pos);
      }
      return;
    }

    // TODO(adonovan): cleanup: replace break after setToken with return,
    // and eliminate null-check of this.kind.
    kind = null;
    while (pos < buffer.length) {
      if (tokenizeTwoChars()) {
        pos += 2;
        return;
      }
      char c = buffer[pos];
      pos++;
      switch (c) {
        case '{':
          setToken(TokenKind.LBRACE, pos - 1, pos);
          openParenStackDepth++;
          break;
        case '}':
          setToken(TokenKind.RBRACE, pos - 1, pos);
          popParen();
          break;
        case '(':
          setToken(TokenKind.LPAREN, pos - 1, pos);
          openParenStackDepth++;
          break;
        case ')':
          setToken(TokenKind.RPAREN, pos - 1, pos);
          popParen();
          break;
        case '[':
          setToken(TokenKind.LBRACKET, pos - 1, pos);
          openParenStackDepth++;
          break;
        case ']':
          setToken(TokenKind.RBRACKET, pos - 1, pos);
          popParen();
          break;
        case '>':
          if (peek(0) == '>' && peek(1) == '=') {
            setToken(TokenKind.GREATER_GREATER_EQUALS, pos - 1, pos + 2);
            pos += 2;
          } else if (peek(0) == '>') {