Method tokenize

ij/src/main/java/ij/macro/Tokenizer.java:16–36

Uses a StreamTokenizer to convert an ImageJ macro file into a token stream.

(String program)

    /** Uses a StreamTokenizer to convert an ImageJ macro file into a token stream. */
    public Program tokenize(String program) {
        if (program.contains("/*") && program.contains("*/"))
            program = addSpacesToEmptyLines(program);
        st = new StreamTokenizer(new StringReader(program));
        st.ordinaryChar('-');
        st.ordinaryChar('/');
        st.ordinaryChar('.');
        st.wordChars('_', '_');
        st.whitespaceChars(128, 255);
        st.slashStarComments(true);
        st.slashSlashComments(true);
        pgm = new Program();
        do {
            getToken();
            addToken();
        } while (token!=EOF);
        if (pgm.hasFunctions)
        	addUserFunctions();
		//IJ.log(program.length()+" "+pgm.getSize()+" "+IJ.d2s((double)program.length()/pgm.getSize(),1)+" "+program.length()/10);
        return pgm;
    }

    final void getToken() {
        try {