(texts: Sequence[Any])
| 162 | |
| 163 | |
| 164 | def build_idf_from_texts(texts: Sequence[Any]) -> Dict[str, float]: |
| 165 | doc_frequency: Counter[str] = Counter() |
| 166 | doc_count = 0 |
| 167 | for text in texts: |
| 168 | unique_tokens = set(tokenize(text)) |
| 169 | if not unique_tokens: |
| 170 | continue |
| 171 | doc_count += 1 |
| 172 | doc_frequency.update(unique_tokens) |
| 173 | if doc_count <= 0: |
| 174 | return {} |
| 175 | return { |
| 176 | token: math.log((doc_count + 1.0) / (freq + 1.0)) + 1.0 |
| 177 | for token, freq in doc_frequency.items() |
| 178 | } |
| 179 | |
| 180 | |
| 181 | def vectorize_text(text: Any, idf: Dict[str, float]) -> Dict[str, float]: |
no test coverage detected