Method docDiff

src/org/antlr/codebuff/Dbg.java:216–251

Compute a document difference metric 0-1.0 between two documents that are identical other than (likely) the whitespace and comments. 1.0 means the docs are maximally different and 0 means docs are identical. The Levenshtein distance between the docs counts only whitespace diffs as the non-WS c

(String original,
	                             String formatted,
	                             Class<? extends Lexer> lexerClass)

	 *  mismatched whitespace? real text are like anchors.
	 */
	public static double docDiff(String original,
	                             String formatted,
	                             Class<? extends Lexer> lexerClass)
		throws Exception
	{
		// Grammar must strip all but real tokens and whitespace (and put that on hidden channel)
		CodeBuffTokenStream original_tokens = Tool.tokenize(original, lexerClass);
//		String s = original_tokens.getText();
		CodeBuffTokenStream formatted_tokens = Tool.tokenize(formatted, lexerClass);
//		String t = formatted_tokens.getText();

		// walk token streams and examine whitespace in between tokens
		int i = -1;
		int ws_distance = 0;
		int original_ws = 0;
		int formatted_ws = 0;
		while ( true ) {
			Token ot = original_tokens.LT(i); // TODO: FIX THIS! can't use LT()
			if ( ot==null || ot.getType()==Token.EOF ) break;
			List<Token> ows = original_tokens.getHiddenTokensToLeft(ot.getTokenIndex());
			original_ws += tokenText(ows).length();

			Token ft = formatted_tokens.LT(i); // TODO: FIX THIS! can't use LT()
			if ( ft==null || ft.getType()==Token.EOF ) break;
			List<Token> fws = formatted_tokens.getHiddenTokensToLeft(ft.getTokenIndex());
			formatted_ws += tokenText(fws).length();

			ws_distance += whitespaceEditDistance(tokenText(ows), tokenText(fws));
			i++;
		}
		// it's probably ok to ignore ws diffs after last real token

		int max_ws = Math.max(original_ws, formatted_ws);
		double normalized_ws_distance = ((float) ws_distance)/max_ws;
		return normalized_ws_distance;
	}

	/** Compare an input document's original text with its formatted output
	 *  and return the ratio of the incorrectWhiteSpaceCount to total whitespace