Evaluate the Vicinity instance on the given query vectors. Computes recall and measures QPS (Queries Per Second). For recall calculation, the same methodology is used as in the ann-benchmarks repository. NOTE: this is only supported for Cosine and Euclidean metric
(
self,
full_vectors: npt.NDArray,
query_vectors: npt.NDArray,
k: int = 10,
epsilon: float = 1e-3,
)
| 326 | return Vicinity(items, backend, metadata=config["metadata"], vector_store=vector_store) |
| 327 | |
| 328 | def evaluate( |
| 329 | self, |
| 330 | full_vectors: npt.NDArray, |
| 331 | query_vectors: npt.NDArray, |
| 332 | k: int = 10, |
| 333 | epsilon: float = 1e-3, |
| 334 | ) -> tuple[float, float]: |
| 335 | """ |
| 336 | Evaluate the Vicinity instance on the given query vectors. |
| 337 | |
| 338 | Computes recall and measures QPS (Queries Per Second). |
| 339 | For recall calculation, the same methodology is used as in the ann-benchmarks repository. |
| 340 | |
| 341 | NOTE: this is only supported for Cosine and Euclidean metric backends. |
| 342 | |
| 343 | :param full_vectors: The full dataset vectors used to build the index. |
| 344 | :param query_vectors: The query vectors to evaluate. |
| 345 | :param k: The number of nearest neighbors to retrieve. |
| 346 | :param epsilon: The epsilon threshold for recall calculation. |
| 347 | :return: A tuple of (QPS, recall). |
| 348 | :raises ValueError: If the metric is not supported by the BasicBackend. |
| 349 | """ |
| 350 | try: |
| 351 | # Validate and map the metric using Metric.from_string |
| 352 | metric_enum = Metric.from_string(self.metric) |
| 353 | if metric_enum not in BasicBackend.supported_metrics: |
| 354 | raise ValueError(f"Unsupported metric '{metric_enum.value}' for BasicBackend.") |
| 355 | basic_metric = metric_enum.value |
| 356 | except ValueError as e: |
| 357 | raise ValueError( |
| 358 | f"Unsupported metric '{self.metric}' for evaluation with BasicBackend. " |
| 359 | f"Supported metrics are: {[m.value for m in BasicBackend.supported_metrics]}" |
| 360 | ) from e |
| 361 | |
| 362 | # Create ground truth Vicinity instance |
| 363 | gt_vicinity = Vicinity.from_vectors_and_items( |
| 364 | vectors=full_vectors, |
| 365 | items=self.items, |
| 366 | backend_type=Backend.BASIC, |
| 367 | metric=basic_metric, |
| 368 | ) |
| 369 | |
| 370 | # Compute ground truth results |
| 371 | gt_distances = [[dist for _, dist in neighbors] for neighbors in gt_vicinity.query(query_vectors, k=k)] |
| 372 | |
| 373 | # Start timer for approximate query |
| 374 | start_time = perf_counter() |
| 375 | run_results = self.query(query_vectors, k=k) |
| 376 | elapsed_time = perf_counter() - start_time |
| 377 | |
| 378 | # Compute QPS |
| 379 | num_queries = len(query_vectors) |
| 380 | qps = num_queries / elapsed_time if elapsed_time > 0 else float("inf") |
| 381 | |
| 382 | # Extract approximate distances |
| 383 | approx_distances = [[dist for _, dist in neighbors] for neighbors in run_results] |
| 384 | |
| 385 | # Compute recall using the ground truth and approximate distances |