Batch evaluation entry point
(self, benchmark_name: str, results: List[Dict[str, str]], concurrency: int = 10)
| 96 | return benchmark_config |
| 97 | |
| 98 | async def __call__(self, benchmark_name: str, results: List[Dict[str, str]], concurrency: int = 10) -> float: |
| 99 | """Batch evaluation entry point""" |
| 100 | benchmark = await self.get(benchmark_name) |
| 101 | if not benchmark: |
| 102 | raise RuntimeError(f"Benchmark {benchmark_name} not initialized.") |
| 103 | |
| 104 | import asyncio |
| 105 | sem = asyncio.Semaphore(concurrency) |
| 106 | |
| 107 | async def _safe_eval(res): |
| 108 | async with sem: |
| 109 | t_id = str(res.get("task_id", "")) |
| 110 | pred = res.get("prediction", "") |
| 111 | gt = res.get("ground_truth") |
| 112 | |
| 113 | if not t_id: |
| 114 | return 0.0 |
| 115 | |
| 116 | try: |
| 117 | # Create a Task object for evaluation |
| 118 | task = Task( |
| 119 | task_id=t_id, |
| 120 | result=pred, |
| 121 | ground_truth=gt |
| 122 | ) |
| 123 | evaluated_task = await benchmark.eval(task) |
| 124 | return evaluated_task.score if evaluated_task else 0.0 |
| 125 | except Exception as e: |
| 126 | logger.error(f"| ❌ Eval failed for task {t_id}: {e}") |
| 127 | return 0.0 |
| 128 | |
| 129 | tasks = [_safe_eval(res) for res in results] |
| 130 | logger.info(f"| 🚀 Starting batch evaluation for {len(tasks)} items in benchmark '{benchmark_name}'") |
| 131 | scores = await asyncio.gather(*tasks) |
| 132 | |
| 133 | avg_score = sum(scores) / len(scores) if scores else 0.0 |
| 134 | logger.info(f"| ✅ Batch evaluation for '{benchmark_name}' completed. Avg score: {avg_score:.4f}") |
| 135 | return avg_score |
| 136 | |
| 137 | async def cleanup(self): |
| 138 | """Cleanup all benchmarks using context manager.""" |