Method tokenize

src/org/apache/pig/parser/QueryParserDriver.java:211–224

(String query, String source)

    }

    static CommonTokenStream tokenize(String query, String source)
            throws ParserException {
        CharStream input;
        try {
            input = new QueryParserStringStream(query, source);
        } catch (IOException ex) {
            throw new ParserException("Unexpected IOException: "
                    + ex.getMessage());
        }
        QueryLexer lexer = new QueryLexer(input);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        checkError(lexer);
        return tokens;
    }

    private static void checkError(BaseRecognizer recognizer)
            throws ParserException {