Parse a code string and collect features with a CollectTokenFeatures. Returns (tokens:list, inject_newlines:boolean[], indent:int[], whitespace:int[], features:list )
(code)
| 47 | |
| 48 | |
| 49 | def extract_data(code): |
| 50 | """ |
| 51 | Parse a code string and collect features with a CollectTokenFeatures. |
| 52 | Returns (tokens:list, inject_newlines:boolean[], indent:int[], |
| 53 | whitespace:int[], features:list<object[]>) |
| 54 | """ |
| 55 | input = InputStream(code) |
| 56 | lexer = JavaLexer(input) |
| 57 | stream = CommonTokenStream(lexer) |
| 58 | parser = JavaParser(stream) |
| 59 | tree = parser.compilationUnit() |
| 60 | collector = CollectTokenFeatures(stream) |
| 61 | walker = ParseTreeWalker() |
| 62 | walker.walk(collector, tree) |
| 63 | return (stream.tokens, collector.inject_newlines, |
| 64 | collector.indent, collector.whitespace, collector.features) |
| 65 | |
| 66 | |
| 67 | def earliestAncestorStartingAtToken(node, token): |
no test coverage detected