Method information_gain

pattern/vector/__init__.py:1134–1174

Returns the information gain (IG) for the given feature, by examining how much it contributes to each document type (class). High IG means low entropy (or predictability) = interesting for feature selection.

(self, word)

    reduce = latent_semantic_analysis
    
    def information_gain(self, word):
        """ Returns the information gain (IG) for the given feature, 
            by examining how much it contributes to each document type (class).
            High IG means low entropy (or predictability) = interesting for feature selection.
        """
        if not self._ig:
            # Based on Vincent Van Asch, http://www.clips.ua.ac.be/~vincent/scripts/textgain.py
            # For classes {xi...xn} and features {yi...yn}:
            # IG(X,Y)  = H(X) - H(X|Y)
            # H(X)     = -sum(p(x) * log2(x) for x in X)
            # H(X|Y)   =  sum(p(y) * H(X|Y=y) for y in Y)
            # H(X|Y=y) = -sum(p(x) * log2(x) for x in X if y in x)
            # H is the entropy for a list of probabilities.
            # Lower entropy indicates predictability, i.e., some values are more probable.
            # H([0.50,0.50]) = 1.00
            # H([0.75,0.25]) = 0.81
            H = entropy
            # X = document type (class) distribution.
            # "How many documents have class xi?"
            X = dict.fromkeys(self.classes, 0)
            for d in self.documents:
                X[d.type] += 1
            # Y = document feature distribution.
            # "How many documents have feature yi?"
            Y = dict.fromkeys(self.features, 0)
            for d in self.documents:
                for y, v in d.vector.items():
                    if v > 0:
                        Y[y] += 1 # Discrete: feature is present (1) or not (0).
            Y = dict((y, Y[y] / float(len(self.documents))) for y in Y)
            # XY = features by class distribution.
            # "How many documents of class xi have feature yi?"
            XY = dict.fromkeys(self.features, {})
            for d in self.documents:
                for y, v in d.vector.items():
                    if v != 0:
                        XY[y][d.type] = XY[y].get(d.type, 0) + 1
            # IG.
            for y in self.features:
                self._ig[y] = H(X.values()) - Y[y] * H(XY[y].values())
        return self._ig[word]
            
    IG = ig = infogain = gain = information_gain