Method __init__

pattern/vector/__init__.py:1270–1316

Latent Semantic Analysis is a statistical machine learning method based on singular value decomposition (SVD), and related to principal component analysis (PCA). Closely related features (words) in the model are combined into "concepts". Documents then get a con

(self, model, k=NORM)

class LSA(object):
    
    def __init__(self, model, k=NORM):
        """ Latent Semantic Analysis is a statistical machine learning method based on 
            singular value decomposition (SVD), and related to principal component analysis (PCA).
            Closely related features (words) in the model are combined into "concepts".
            Documents then get a concept vector that is an approximation of the original vector,
            but with reduced dimensionality so that cosine similarity and clustering run faster.
        """
        import numpy
        # Calling Model.vector() in a loop is quite slow, we should refactor this:
        matrix = [model.vector(d).values() for d in model.documents]
        matrix = numpy.array(matrix)
        # Singular value decomposition, where u * sigma * vt = svd(matrix).
        # Sigma is the diagonal matrix of singular values,
        # u has document rows and concept columns, vt has concept rows and term columns.
        u, sigma, vt = numpy.linalg.svd(matrix, full_matrices=False)
        # Delete the smallest coefficients in the diagonal matrix (i.e., at the end of the list).
        # The difficulty and weakness of LSA is knowing how many dimensions to reduce
        # (generally L2-norm is used).
        if k == L1:
            k = int(round(numpy.linalg.norm(sigma, 1)))
        if k == L2 or k == NORM:
            k = int(round(numpy.linalg.norm(sigma, 2)))
        if k == TOP300:
            k = max(0, len(sigma) - 300)
        if isinstance(k, int):
            k = max(0, len(sigma) - k)
        if type(k).__name__ == "function":
            k = max(0, int(k(sigma)))
        #print numpy.dot(u, numpy.dot(numpy.diag(sigma), vt))
        # Apply dimension reduction.
        # The maximum length of a concept vector = the number of documents.
        assert k < len(model.documents), \
            "can't create more dimensions than there are documents"
        tail = lambda list, i: range(len(list)-i, len(list))
        u, sigma, vt = (
            numpy.delete(u, tail(u[0], k), axis=1),
            numpy.delete(sigma, tail(sigma, k), axis=0),
            numpy.delete(vt, tail(vt, k), axis=0)
        )
        # Store as Python dict and lists so we can pickle it.
        self.model = model
        self._terms = dict(enumerate(model.vector().keys())) # Vt-index => word.
        self.u, self.sigma, self.vt = (
            dict((d.id, Vector((i, float(x)) for i, x in enumerate(v))) for d, v in izip(model, u)),
            list(sigma),
            [[float(x) for x in v] for v in vt]
        )
    
    @property
    def terms(self):