Load a vocab file .tokens and return mapping.
()
| 34 | |
| 35 | /** Load a vocab file {@code <vocabName>.tokens} and return mapping. */ |
| 36 | public Map<String,Integer> load() { |
| 37 | Map<String,Integer> tokens = new LinkedHashMap<String,Integer>(); |
| 38 | int maxTokenType = -1; |
| 39 | File fullFile = getImportedVocabFile(); |
| 40 | FileInputStream fis = null; |
| 41 | BufferedReader br = null; |
| 42 | Tool tool = g.tool; |
| 43 | String vocabName = g.getOptionString("tokenVocab"); |
| 44 | try { |
| 45 | Pattern tokenDefPattern = Pattern.compile("([^\n]+?)[ \\t]*?=[ \\t]*?([0-9]+)"); |
| 46 | fis = new FileInputStream(fullFile); |
| 47 | InputStreamReader isr; |
| 48 | if (tool.grammarEncoding != null) { |
| 49 | isr = new InputStreamReader(fis, tool.grammarEncoding); |
| 50 | } |
| 51 | else { |
| 52 | isr = new InputStreamReader(fis); |
| 53 | } |
| 54 | |
| 55 | br = new BufferedReader(isr); |
| 56 | String tokenDef = br.readLine(); |
| 57 | int lineNum = 1; |
| 58 | while ( tokenDef!=null ) { |
| 59 | Matcher matcher = tokenDefPattern.matcher(tokenDef); |
| 60 | if ( matcher.find() ) { |
| 61 | String tokenID = matcher.group(1); |
| 62 | String tokenTypeS = matcher.group(2); |
| 63 | int tokenType; |
| 64 | try { |
| 65 | tokenType = Integer.valueOf(tokenTypeS); |
| 66 | } |
| 67 | catch (NumberFormatException nfe) { |
| 68 | tool.errMgr.toolError(ErrorType.TOKENS_FILE_SYNTAX_ERROR, |
| 69 | vocabName + CodeGenerator.VOCAB_FILE_EXTENSION, |
| 70 | " bad token type: "+tokenTypeS, |
| 71 | lineNum); |
| 72 | tokenType = Token.INVALID_TOKEN_TYPE; |
| 73 | } |
| 74 | tool.log("grammar", "import "+tokenID+"="+tokenType); |
| 75 | tokens.put(tokenID, tokenType); |
| 76 | maxTokenType = Math.max(maxTokenType,tokenType); |
| 77 | lineNum++; |
| 78 | } |
| 79 | else { |
| 80 | if ( tokenDef.length()>0 ) { // ignore blank lines |
| 81 | tool.errMgr.toolError(ErrorType.TOKENS_FILE_SYNTAX_ERROR, |
| 82 | vocabName + CodeGenerator.VOCAB_FILE_EXTENSION, |
| 83 | " bad token def: " + tokenDef, |
| 84 | lineNum); |
| 85 | } |
| 86 | } |
| 87 | tokenDef = br.readLine(); |
| 88 | } |
| 89 | } |
| 90 | catch (FileNotFoundException fnfe) { |
| 91 | GrammarAST inTree = g.ast.getOptionAST("tokenVocab"); |
| 92 | String inTreeValue = inTree.getToken().getText(); |
| 93 | if ( vocabName.equals(inTreeValue) ) { |
no test coverage detected