Build sparse queries for brute-force comparison
(queries: Dict, dataset: str, corpus_stats)
| 653 | |
| 654 | |
| 655 | def build_sparse_queries_for_bf(queries: Dict, dataset: str, corpus_stats) -> List[Dict]: |
| 656 | """Build sparse queries for brute-force comparison""" |
| 657 | cache_file = Path("datasets") / f"hybrid_{dataset}" / "sparse_queries.pkl" |
| 658 | if cache_file.exists(): |
| 659 | return pickle.loads(cache_file.read_bytes()) |
| 660 | |
| 661 | total_docs, term_doc_freq, avg_len = corpus_stats |
| 662 | stemmer = SnowballStemmer("english") |
| 663 | query_vecs = [] |
| 664 | for qid, qtext in tqdm(queries.items(), desc="Queries"): |
| 665 | tokens = SimpleTokenizer.tokenize(qtext) |
| 666 | terms = [ |
| 667 | stemmer.stem_word(t.lower()) |
| 668 | for t in tokens |
| 669 | if t.lower() not in STOPWORDS and t not in PUNCT and len(t) <= 40 |
| 670 | ] |
| 671 | tf = defaultdict(int) |
| 672 | for tok in terms: |
| 673 | tf[tok] += 1 |
| 674 | indices, values = [], [] |
| 675 | for tok, raw in tf.items(): |
| 676 | if tok in term_doc_freq: |
| 677 | idf = compute_bm25_idf(total_docs, term_doc_freq[tok]) |
| 678 | tf_score = compute_bm25_tf(raw, len(terms), avg_len) |
| 679 | bm25_score = idf * tf_score |
| 680 | if bm25_score > 0: |
| 681 | indices.append(hash(tok) % (2**31)) # Simple hash for index |
| 682 | values.append(bm25_score) |
| 683 | query_vecs.append( |
| 684 | {"id": qid, "text": " ".join(terms), "indices": indices, "values": values} |
| 685 | ) |
| 686 | |
| 687 | cache_file.parent.mkdir(parents=True, exist_ok=True) |
| 688 | cache_file.write_bytes(pickle.dumps(query_vecs)) |
| 689 | return query_vecs |
| 690 | |
| 691 | |
| 692 | def brute_force_sparse( |
no test coverage detected