| 916 | return results |
| 917 | |
| 918 | def evaluate( |
| 919 | hybrid_results: List[List[Dict]], |
| 920 | bf_dense: List[Dict], |
| 921 | bf_sparse: List[Dict], |
| 922 | query_ids: List[str], |
| 923 | qrels: Dict, |
| 924 | ) -> Dict: |
| 925 | # Hybrid vs dense brute-force recall |
| 926 | hybrid_dict = { |
| 927 | qid: {r["id"]: r["score"] for r in res} |
| 928 | for qid, res in zip(query_ids, hybrid_results) |
| 929 | } |
| 930 | bf_dense_dict = { |
| 931 | r["query_id"]: {rr["id"]: rr["score"] for rr in r["top_results"]} |
| 932 | for r in bf_dense |
| 933 | } |
| 934 | recalls_dense = [] |
| 935 | for qid, bf_ids in bf_dense_dict.items(): |
| 936 | srv_ids = set(hybrid_dict.get(qid, {})) |
| 937 | recalls_dense.append(len(srv_ids & set(bf_ids)) / len(bf_ids) if bf_ids else 0) |
| 938 | bf_recall_dense = sum(recalls_dense) / len(recalls_dense) if recalls_dense else 0 |
| 939 | |
| 940 | # Hybrid vs sparse brute-force recall |
| 941 | bf_sparse_dict = { |
| 942 | r["query_id"]: {rr["id"]: rr["score"] for rr in r["top_results"]} |
| 943 | for r in bf_sparse |
| 944 | } |
| 945 | recalls_sparse = [] |
| 946 | for qid, bf_ids in bf_sparse_dict.items(): |
| 947 | srv_ids = set(hybrid_dict.get(qid, {})) |
| 948 | recalls_sparse.append(len(srv_ids & set(bf_ids)) / len(bf_ids) if bf_ids else 0) |
| 949 | |
| 950 | bf_recall_sparse = ( |
| 951 | sum(recalls_sparse) / len(recalls_sparse) if recalls_sparse else 0 |
| 952 | ) |
| 953 | |
| 954 | # BEIR metrics (using hybrid results) |
| 955 | try: |
| 956 | evaluator = EvaluateRetrieval(k_values=[1, 5, 10]) |
| 957 | ndcg, _map, recall, precision = evaluator.evaluate(qrels, hybrid_dict, [1, 5, 10]) |
| 958 | beir_metrics = { |
| 959 | "ndcg@10": ndcg["NDCG@10"], |
| 960 | "beir_recall@10": recall["Recall@10"], |
| 961 | "map": _map["MAP@10"], |
| 962 | "p@1": precision["P@1"], |
| 963 | "p@5": precision["P@5"], |
| 964 | "p@10": precision["P@10"], |
| 965 | } |
| 966 | except Exception as e: |
| 967 | print(f"BEIR evaluation failed: {e}") |
| 968 | beir_metrics = { |
| 969 | "ndcg@10": 0.0, |
| 970 | "beir_recall@10": 0.0, |
| 971 | "map": 0.0, |
| 972 | "p@1": 0.0, |
| 973 | "p@5": 0.0, |
| 974 | "p@10": 0.0, |
| 975 | } |