Clustering is an unsupervised machine learning method for grouping similar documents. - k-means clustering returns a list of k clusters (each is a list of documents). - hierarchical clustering returns a list of documents and Cluster objects, where a Cluster is
(self, documents=ALL, method=KMEANS, **kwargs)
| 1085 | # def cluster(self, documents=ALL, method=KMEANS, k=10, iterations=10) |
| 1086 | # def cluster(self, documents=ALL, method=HIERARCHICAL, k=1, iterations=1000) |
| 1087 | def cluster(self, documents=ALL, method=KMEANS, **kwargs): |
| 1088 | """ Clustering is an unsupervised machine learning method for grouping similar documents. |
| 1089 | - k-means clustering returns a list of k clusters (each is a list of documents). |
| 1090 | - hierarchical clustering returns a list of documents and Cluster objects, |
| 1091 | where a Cluster is a list of documents and other clusters (see Cluster.flatten()). |
| 1092 | """ |
| 1093 | if documents == ALL: |
| 1094 | documents = self.documents |
| 1095 | if not getattr(self, "lsa", None): |
| 1096 | # Using document vectors: |
| 1097 | vectors, features = [d.vector for d in documents], self.vector.keys() |
| 1098 | else: |
| 1099 | # Using LSA concept space: |
| 1100 | vectors, features = [self.lsa[d.id] for d in documents], range(len(self.lsa)) |
| 1101 | # Create a dictionary of vector.id => Document. |
| 1102 | # We need it to map the clustered vectors back to the actual documents. |
| 1103 | map = dict((v.id, documents[i]) for i, v in enumerate(vectors)) |
| 1104 | if method in (KMEANS, "kmeans"): |
| 1105 | clusters = k_means(vectors, |
| 1106 | k = kwargs.pop("k", 10), |
| 1107 | iterations = kwargs.pop("iterations", 10), |
| 1108 | features = features, **kwargs) |
| 1109 | if method == HIERARCHICAL: |
| 1110 | clusters = hierarchical(vectors, |
| 1111 | k = kwargs.pop("k", 1), |
| 1112 | iterations = kwargs.pop("iterations", 1000), |
| 1113 | features = features, **kwargs) |
| 1114 | if method in (KMEANS, "kmeans"): |
| 1115 | clusters = [[map[v.id] for v in cluster] for cluster in clusters] |
| 1116 | if method == HIERARCHICAL: |
| 1117 | clusters.traverse(visit=lambda cluster: \ |
| 1118 | [cluster.__setitem__(i, map[v.id]) |
| 1119 | for i, v in enumerate(cluster) if not isinstance(v, Cluster)]) |
| 1120 | return clusters |
| 1121 | |
| 1122 | def latent_semantic_analysis(self, dimensions=NORM): |
| 1123 | """ Creates LSA concept vectors by reducing the vector space's dimensionality. |