(rows: Sequence[Dict[str, Any]])
| 254 | |
| 255 | |
| 256 | def build_bm25_state(rows: Sequence[Dict[str, Any]]) -> Dict[str, Any]: |
| 257 | docs = [tokenize(paper_text(row)) for row in rows] |
| 258 | doc_freq: Counter[str] = Counter() |
| 259 | for tokens in docs: |
| 260 | doc_freq.update(set(tokens)) |
| 261 | doc_count = len(docs) |
| 262 | avg_len = sum(len(tokens) for tokens in docs) / max(1, doc_count) |
| 263 | idf = { |
| 264 | token: math.log(1.0 + (doc_count - freq + 0.5) / (freq + 0.5)) |
| 265 | for token, freq in doc_freq.items() |
| 266 | } |
| 267 | return {"docs": docs, "idf": idf, "avg_len": avg_len} |
| 268 | |
| 269 | |
| 270 | def bm25_score( |
no test coverage detected