Returns the document frequency for the given word or feature. Returns 0 if there are no documents in the model (e.g. no word frequency). df = number of documents containing the word / number of documents. The more occurences of the word across the model, the high
(self, word)
| 939 | return self._index[name] |
| 940 | |
| 941 | def document_frequency(self, word): |
| 942 | """ Returns the document frequency for the given word or feature. |
| 943 | Returns 0 if there are no documents in the model (e.g. no word frequency). |
| 944 | df = number of documents containing the word / number of documents. |
| 945 | The more occurences of the word across the model, the higher its df weight. |
| 946 | """ |
| 947 | if len(self.documents) == 0: |
| 948 | return 0.0 |
| 949 | if len(self._df) == 0: |
| 950 | # Caching document frequency for each word gives a 300x performance boost |
| 951 | # (i.e., calculated all at once). Drawback is if you need it for just one word. |
| 952 | df = self._df |
| 953 | for d in self.documents: |
| 954 | for w, f in d.terms.iteritems(): |
| 955 | if f != 0: |
| 956 | df[w] = (w in df) and df[w] + 1 or 1.0 |
| 957 | for w in df: |
| 958 | df[w] /= float(len(self.documents)) |
| 959 | return self._df.get(word, 0.0) |
| 960 | |
| 961 | df = document_frequency |
| 962 |