Method examineCorpus

src/org/antlr/codebuff/validation/Entropy.java:158–189

(Corpus corpus)

	}

	public static void examineCorpus(Corpus corpus) {
		ListMultimap<FeatureVectorAsObject, Integer> wsByFeatureVectorGroup = ArrayListMultimap.create();
		ListMultimap<FeatureVectorAsObject, Integer> hposByFeatureVectorGroup = ArrayListMultimap.create();
		int numContexts = corpus.featureVectors.size();
		for (int i = 0; i<numContexts; i++) {
			int[] X = corpus.featureVectors.get(i);
			int y1  = corpus.injectWhitespace.get(i);
			int y2  = corpus.hpos.get(i);
			wsByFeatureVectorGroup.put(new FeatureVectorAsObject(X, Trainer.FEATURES_INJECT_WS), y1);
			hposByFeatureVectorGroup.put(new FeatureVectorAsObject(X, Trainer.FEATURES_HPOS), y2);
		}
		List<Double> wsEntropies = new ArrayList<>();
		List<Double> hposEntropies = new ArrayList<>();
		for (FeatureVectorAsObject x : wsByFeatureVectorGroup.keySet()) {
			List<Integer> cats = wsByFeatureVectorGroup.get(x);
			List<Integer> cats2 = hposByFeatureVectorGroup.get(x);
			HashBag<Integer> wsCats = getCategoriesBag(cats);
			HashBag<Integer> hposCats = getCategoriesBag(cats2);
			double wsEntropy = getNormalizedCategoryEntropy(getCategoryRatios(wsCats.values()));
			double hposEntropy = getNormalizedCategoryEntropy(getCategoryRatios(hposCats.values()));
			wsEntropies.add(wsEntropy);
			hposEntropies.add(hposEntropy);
			System.out.printf("%130s : %s,%s %s,%s\n", x,
			                  wsCats, wsEntropy,
			                  hposCats, hposEntropy);
		}
		System.out.println("MEAN "+mean(wsEntropies));
		System.out.println("MEAN "+mean(hposEntropies));
		float contextRichness = wsEntropies.size()/(float) numContexts; // 0..1 where 1 means every token had different context
		System.out.println("Context richness = "+contextRichness+
			                   " uniq ctxs="+wsEntropies.size()+", nctxs="+numContexts);
	}

	/** Return diversity index, e^entropy.
	 *  https://en.wikipedia.org/wiki/Diversity_index