(doc)
| 621 | term_doc_freq[t] += 1 |
| 622 | |
| 623 | def build_vector(doc): |
| 624 | tf = defaultdict(int) |
| 625 | for tok in doc["tokens"]: |
| 626 | tf[tok] += 1 |
| 627 | indices, values = [], [] |
| 628 | for tok, raw in tf.items(): |
| 629 | idf = compute_bm25_idf(total_docs, term_doc_freq[tok]) |
| 630 | tf_score = compute_bm25_tf(raw, doc["length"], avg_len) |
| 631 | bm25_score = idf * tf_score |
| 632 | if bm25_score > 0: |
| 633 | indices.append(hash(tok) % (2**31)) # Simple hash for index |
| 634 | values.append(bm25_score) |
| 635 | return { |
| 636 | "id": doc["id"], |
| 637 | "text": " ".join(doc["tokens"]), |
| 638 | "indices": indices, |
| 639 | "values": values, |
| 640 | } |
| 641 | |
| 642 | vectors = [] |
| 643 | with ThreadPoolExecutor(max_workers=MAX_WORKERS) as ex: |
nothing calls this directly
no test coverage detected