| 167 | } |
| 168 | |
| 169 | public float getWSEditDistance() throws Exception { |
| 170 | List<Token> wsTokens = filter(originalTokens.getTokens(), |
| 171 | t -> t.getText().matches("\\s+")); // only count whitespace |
| 172 | String originalWS = tokenText(wsTokens); |
| 173 | |
| 174 | String formattedOutput = getOutput(); |
| 175 | CommonTokenStream formatted_tokens = tokenize(formattedOutput, corpus.language.lexerClass); |
| 176 | wsTokens = filter(formatted_tokens.getTokens(), |
| 177 | t -> t.getText().matches("\\s+")); |
| 178 | String formattedWS = tokenText(wsTokens); |
| 179 | |
| 180 | float editDistance = normalizedLevenshteinDistance(originalWS, formattedWS); |
| 181 | return editDistance; |
| 182 | } |
| 183 | |
| 184 | public void processToken(int indexIntoRealTokens, int tokenIndexInStream, boolean collectAnalysis) { |
| 185 | CommonToken curToken = (CommonToken)testDoc.tokens.get(tokenIndexInStream); |