Calculate recall metrics
(brute_force_results, server_results, top_k=10)
| 376 | return [[ind, val] for ind, val in zip(vector["indices"], vector["values"])] |
| 377 | |
| 378 | def calculate_recall(brute_force_results, server_results, top_k=10): |
| 379 | """Calculate recall metrics""" |
| 380 | recalls = [] |
| 381 | |
| 382 | for i, (bf_result, server_result) in enumerate( |
| 383 | zip(brute_force_results, server_results) |
| 384 | ): |
| 385 | # Debug: Print first few server results to understand structure |
| 386 | if i < 3: |
| 387 | print(f"\nDebug - Query {i}:") |
| 388 | print(f"Brute force top 3: {bf_result['top_results'][:3]}") |
| 389 | print(f"Server result keys: {list(server_result.keys())}") |
| 390 | print(f"Server result: {server_result}") |
| 391 | |
| 392 | bf_ids = set(item["id"] for item in bf_result["top_results"]) |
| 393 | |
| 394 | # Handle different possible server response structures |
| 395 | if "results" in server_result: |
| 396 | server_items = server_result["results"] |
| 397 | elif "vectors" in server_result: |
| 398 | server_items = server_result["vectors"] |
| 399 | else: |
| 400 | print(f"Warning: Unexpected server result structure: {server_result}") |
| 401 | server_items = [] |
| 402 | |
| 403 | # Extract IDs from server results - handle different possible structures |
| 404 | server_ids = set() |
| 405 | for item in server_items: |
| 406 | if isinstance(item, dict): |
| 407 | if "id" in item: |
| 408 | server_ids.add(item["id"]) |
| 409 | elif "vector_id" in item: |
| 410 | server_ids.add(item["vector_id"]) |
| 411 | else: |
| 412 | print(f"Warning: Unknown server item structure: {item}") |
| 413 | else: |
| 414 | print(f"Warning: Server item is not a dict: {item}") |
| 415 | |
| 416 | if not bf_ids: |
| 417 | continue # Skip if brute force found no results |
| 418 | |
| 419 | intersection = bf_ids.intersection(server_ids) |
| 420 | recall = len(intersection) / len(bf_ids) |
| 421 | recalls.append(recall) |
| 422 | |
| 423 | # Debug: Print recall for first few queries |
| 424 | if i < 3: |
| 425 | print(f"BF IDs (first 5): {list(bf_ids)[:5]}") |
| 426 | print(f"Server IDs (first 5): {list(server_ids)[:5]}") |
| 427 | print(f"Intersection: {len(intersection)}") |
| 428 | print(f"Recall: {recall:.2%}") |
| 429 | |
| 430 | avg_recall = sum(recalls) / len(recalls) if recalls else 0 |
| 431 | return avg_recall, recalls |
| 432 | |
| 433 | |
| 434 | def run_rps_tests( |