(InputDocument doc, int tokenIndexInStream)
| 344 | } |
| 345 | |
| 346 | public int[] getFeatures(InputDocument doc, int tokenIndexInStream) { |
| 347 | Token prevToken = doc.tokens.getPreviousRealToken(tokenIndexInStream); |
| 348 | Token prevPrevToken = prevToken!=null ? doc.tokens.getPreviousRealToken(prevToken.getTokenIndex()) : null; |
| 349 | boolean prevTokenStartsLine = false; |
| 350 | if ( prevToken!=null && prevPrevToken!=null ) { |
| 351 | prevTokenStartsLine = prevToken.getLine()>prevPrevToken.getLine(); |
| 352 | } |
| 353 | TerminalNode node = tokenToNodeMap.get(doc.tokens.get(tokenIndexInStream)); |
| 354 | if ( node==null ) { |
| 355 | System.err.println("### No node associated with token "+doc.tokens.get(tokenIndexInStream)); |
| 356 | return null; |
| 357 | } |
| 358 | |
| 359 | Token curToken = node.getSymbol(); |
| 360 | |
| 361 | boolean curTokenStartsNewLine = false; |
| 362 | if ( prevToken==null ) curTokenStartsNewLine = true; // we must be at start of file |
| 363 | else if ( line > prevToken.getLine() ) curTokenStartsNewLine = true; |
| 364 | |
| 365 | int[] features = getContextFeatures(corpus, tokenToNodeMap, doc, tokenIndexInStream); |
| 366 | |
| 367 | setListInfoFeatures(tokenToListInfo, features, curToken); |
| 368 | |
| 369 | features[INDEX_PREV_FIRST_ON_LINE] = prevTokenStartsLine ? 1 : 0; |
| 370 | features[INDEX_FIRST_ON_LINE] = curTokenStartsNewLine ? 1 : 0; |
| 371 | |
| 372 | return features; |
| 373 | } |
| 374 | |
| 375 | /** Look into the originalTokens stream to get the comments to the left of current |
| 376 | * token. Emit all whitespace and comments except for whitespace at the |
no test coverage detected