Method collectTokenPairsAndSplitListInfo

src/org/antlr/codebuff/Corpus.java:93–140

Walk all documents to compute matching token dependencies (we need this for feature computation) While we're at it, find sibling lists.

()

	 *  While we're at it, find sibling lists.
	 */
	public void collectTokenPairsAndSplitListInfo() throws NoSuchMethodException, InstantiationException, IllegalAccessException, java.lang.reflect.InvocationTargetException {
		Vocabulary vocab = getLexer(language.lexerClass, null).getVocabulary();
		String[] ruleNames = getParser(language.parserClass, null).getRuleNames();
		CollectTokenPairs collectTokenPairs = new CollectTokenPairs(vocab, ruleNames);
		CollectSiblingLists collectSiblingLists = new CollectSiblingLists();
		for (InputDocument doc : documents) {
			collectSiblingLists.setTokens(doc.tokens, doc.tree, doc.tokenToNodeMap);
			ParseTreeWalker.DEFAULT.walk(collectTokenPairs, doc.tree);
			ParseTreeWalker.DEFAULT.walk(collectSiblingLists, doc.tree);
		}
		ruleToPairsBag = collectTokenPairs.getDependencies();
		rootAndChildListStats = collectSiblingLists.getListStats();
		rootAndSplitChildListStats = collectSiblingLists.getSplitListStats();
		tokenToListInfo = collectSiblingLists.getTokenToListInfo();

		if ( false ) {
			for (RuleAltKey ruleAltKey : ruleToPairsBag.keySet()) {
				List<Pair<Integer, Integer>> pairs = ruleToPairsBag.get(ruleAltKey);
				System.out.print(ruleAltKey+" -> ");
				for (Pair<Integer, Integer> p : pairs) {
					System.out.print(vocab.getDisplayName(p.a)+","+vocab.getDisplayName(p.b)+" ");
				}
				System.out.println();
			}
		}

		if ( false ) {
			for (ParentSiblingListKey siblingPairs : rootAndChildListStats.keySet()) {
				String parent = ruleNames[siblingPairs.parentRuleIndex];
				parent = parent.replace("Context","");
				String siblingListName = ruleNames[siblingPairs.childRuleIndex];
				siblingListName = siblingListName.replace("Context","");
				System.out.println(parent+":"+siblingPairs.parentRuleAlt+"->"+siblingListName+":"+siblingPairs.childRuleAlt+
					                   " (n,min,median,var,max)="+rootAndChildListStats.get(siblingPairs));
			}
			Map<ParentSiblingListKey, Integer> splitListForms =
				collectSiblingLists.getSplitListForms();
			for (ParentSiblingListKey siblingPairs : rootAndSplitChildListStats.keySet()) {
				String parent = ruleNames[siblingPairs.parentRuleIndex];
				parent = parent.replace("Context","");
				String siblingListName = ruleNames[siblingPairs.childRuleIndex];
				siblingListName = siblingListName.replace("Context","");
				System.out.println("SPLIT " +parent+":"+siblingPairs.parentRuleAlt+"->"+siblingListName+":"+siblingPairs.childRuleAlt+
					                   " (n,min,median,var,max)="+rootAndSplitChildListStats.get(siblingPairs)+
				                  " form "+splitListForms.get(siblingPairs));
			}
		}
	}

	public void trainOnSampleDocs() throws Exception {
		documentsPerExemplar = new ArrayList<>();