(text: Any, idf: Dict[str, float])
| 179 | |
| 180 | |
| 181 | def vectorize_text(text: Any, idf: Dict[str, float]) -> Dict[str, float]: |
| 182 | counts = Counter(tokenize(text)) |
| 183 | if not counts: |
| 184 | return {} |
| 185 | default_idf = math.log(max(2.0, len(idf) + 1.0)) |
| 186 | vector = { |
| 187 | token: (1.0 + math.log(freq)) * idf.get(token, default_idf) |
| 188 | for token, freq in counts.items() |
| 189 | } |
| 190 | return normalize_vector(vector) |
| 191 | |
| 192 | |
| 193 | def normalize_vector(vector: Dict[str, float]) -> Dict[str, float]: |
no test coverage detected