(text: Any, idf: Dict[str, float])
| 241 | |
| 242 | |
| 243 | def vectorize_text(text: Any, idf: Dict[str, float]) -> Dict[str, float]: |
| 244 | counts = Counter(tokenize(text)) |
| 245 | if not counts: |
| 246 | return {} |
| 247 | default_idf = math.log(max(2.0, len(idf) + 1.0)) |
| 248 | vector = { |
| 249 | token: (1.0 + math.log(freq)) * idf.get(token, default_idf) |
| 250 | for token, freq in counts.items() |
| 251 | } |
| 252 | return normalize_vector(vector) |
| 253 | |
| 254 | |
| 255 | def normalize_vector(vector: Dict[str, float]) -> Dict[str, float]: |
no test coverage detected