Return inject_newlines:boolean[], indent:int[], whitespace:int[], features:list collected from all corpus files found recursively under dir.
(dir)
| 78 | |
| 79 | |
| 80 | def analyze_corpus(dir): |
| 81 | """ |
| 82 | Return inject_newlines:boolean[], indent:int[], whitespace:int[], |
| 83 | features:list<object[]> collected from all corpus files found recursively under dir. |
| 84 | """ |
| 85 | inject_newlines = [] |
| 86 | indents = [] |
| 87 | features = [] |
| 88 | whitespace = [] |
| 89 | for fname in files(dir): |
| 90 | print fname |
| 91 | with open(fname, 'r') as f: |
| 92 | code = f.read() |
| 93 | code = code.expandtabs(TABSIZE) |
| 94 | tokens, nl, indent, ws, predictors = extract_data(code) |
| 95 | indents += indent |
| 96 | inject_newlines += nl |
| 97 | whitespace += ws |
| 98 | features += predictors |
| 99 | return (inject_newlines, indents, whitespace, features) |
| 100 | |
| 101 | |
| 102 | def convert_categorical_data(features): |
no test coverage detected