Method allTokens

src/test/java/net/starlark/java/syntax/LexerTest.java:65–77

(Lexer lexer)

  }

  private ArrayList<Token> allTokens(Lexer lexer) {
    ArrayList<Token> result = new ArrayList<>();
    do {
      lexer.nextToken();
      Token tok = new Token();
      tok.kind = lexer.kind;
      tok.start = lexer.start;
      tok.end = lexer.end;
      tok.value = lexer.value;
      result.add(tok);
    } while (lexer.kind != TokenKind.EOF);
    return result;
  }

  private Token[] tokens(String input) {
    ArrayList<Token> result = allTokens(createLexer(input));