(int indexIntoRealTokens, int tokenIndexInStream, boolean collectAnalysis)
| 182 | } |
| 183 | |
| 184 | public void processToken(int indexIntoRealTokens, int tokenIndexInStream, boolean collectAnalysis) { |
| 185 | CommonToken curToken = (CommonToken)testDoc.tokens.get(tokenIndexInStream); |
| 186 | String tokText = curToken.getText(); |
| 187 | TerminalNode node = tokenToNodeMap.get(curToken); |
| 188 | |
| 189 | int[] features = getFeatures(testDoc, tokenIndexInStream); |
| 190 | int[] featuresForAlign = new int[features.length]; |
| 191 | System.arraycopy(features, 0, featuresForAlign, 0, features.length); |
| 192 | |
| 193 | int injectNL_WS = wsClassifier.classify(k, features, Trainer.MAX_WS_CONTEXT_DIFF_THRESHOLD); |
| 194 | |
| 195 | injectNL_WS = emitCommentsToTheLeft(tokenIndexInStream, injectNL_WS); |
| 196 | |
| 197 | int newlines = 0; |
| 198 | int ws = 0; |
| 199 | if ( (injectNL_WS&0xFF)==CAT_INJECT_NL ) { |
| 200 | newlines = Trainer.unnlcat(injectNL_WS); |
| 201 | } |
| 202 | else if ( (injectNL_WS&0xFF)==CAT_INJECT_WS ) { |
| 203 | ws = Trainer.unwscat(injectNL_WS); |
| 204 | } |
| 205 | |
| 206 | if ( newlines==0 && ws==0 && cannotJoin(realTokens.get(indexIntoRealTokens-1), curToken) ) { // failsafe! |
| 207 | ws = 1; |
| 208 | } |
| 209 | |
| 210 | int alignOrIndent = CAT_ALIGN; |
| 211 | |
| 212 | if ( newlines>0 ) { |
| 213 | output.append(Tool.newlines(newlines)); |
| 214 | line+=newlines; |
| 215 | charPosInLine = 0; |
| 216 | |
| 217 | // getFeatures() doesn't know what line curToken is on. If \n, we need to find exemplars that start a line |
| 218 | featuresForAlign[INDEX_FIRST_ON_LINE] = 1; // use \n prediction to match exemplars for alignment |
| 219 | |
| 220 | alignOrIndent = hposClassifier.classify(k, featuresForAlign, MAX_ALIGN_CONTEXT_DIFF_THRESHOLD); |
| 221 | |
| 222 | if ( (alignOrIndent&0xFF)==CAT_ALIGN_WITH_ANCESTOR_CHILD ) { |
| 223 | align(alignOrIndent, node); |
| 224 | } |
| 225 | else if ( (alignOrIndent&0xFF)==CAT_INDENT_FROM_ANCESTOR_CHILD ) { |
| 226 | indent(alignOrIndent, node); |
| 227 | } |
| 228 | else if ( (alignOrIndent&0xFF)==CAT_ALIGN ) { |
| 229 | List<Token> tokensOnPreviousLine = getTokensOnPreviousLine(testDoc.tokens, tokenIndexInStream, line); |
| 230 | if ( tokensOnPreviousLine.size()>0 ) { |
| 231 | Token firstTokenOnPrevLine = tokensOnPreviousLine.get(0); |
| 232 | int indentCol = firstTokenOnPrevLine.getCharPositionInLine(); |
| 233 | charPosInLine = indentCol; |
| 234 | output.append(Tool.spaces(indentCol)); |
| 235 | } |
| 236 | } |
| 237 | else if ( (alignOrIndent&0xFF)==CAT_INDENT ) { |
| 238 | indent(alignOrIndent, node); |
| 239 | } |
| 240 | } |
| 241 | else { |
no test coverage detected