Tokenize code and then, one token at a time, predict newline or not. Do prediction of newline on the fly, adjusting token line/column.
(newline_forest, indent_forest, whitespace_forest, vec, code)
| 129 | |
| 130 | |
| 131 | def format_code(newline_forest, indent_forest, whitespace_forest, vec, code): |
| 132 | """ |
| 133 | Tokenize code and then, one token at a time, predict newline or not. |
| 134 | Do prediction of newline on the fly, adjusting token line/column. |
| 135 | """ |
| 136 | |
| 137 | # tokenize and wack location info in tokens |
| 138 | input = InputStream(code) |
| 139 | lexer = JavaLexer(input) |
| 140 | stream = CommonTokenStream(lexer) |
| 141 | stream.fill() |
| 142 | # wipe out token location information in sample |
| 143 | for t in stream.tokens: |
| 144 | t.line = 0 |
| 145 | t.column = 0 |
| 146 | |
| 147 | # parse to get parse tree |
| 148 | parser = JavaParser(stream) |
| 149 | tree = parser.compilationUnit() |
| 150 | |
| 151 | # compute feature vector for each token and adjust line/column as we walk tree |
| 152 | collector = ProcessTokens(newline_forest, indent_forest, whitespace_forest, vec, stream) |
| 153 | walker = ParseTreeWalker() |
| 154 | walker.walk(collector, tree) |
| 155 | |
| 156 | |
| 157 | def files(dir): |
no test coverage detected