| 105 | sem = asyncio.Semaphore(concurrency) |
| 106 | |
| 107 | async def _safe_eval(res): |
| 108 | async with sem: |
| 109 | t_id = str(res.get("task_id", "")) |
| 110 | pred = res.get("prediction", "") |
| 111 | gt = res.get("ground_truth") |
| 112 | |
| 113 | if not t_id: |
| 114 | return 0.0 |
| 115 | |
| 116 | try: |
| 117 | # Create a Task object for evaluation |
| 118 | task = Task( |
| 119 | task_id=t_id, |
| 120 | result=pred, |
| 121 | ground_truth=gt |
| 122 | ) |
| 123 | evaluated_task = await benchmark.eval(task) |
| 124 | return evaluated_task.score if evaluated_task else 0.0 |
| 125 | except Exception as e: |
| 126 | logger.error(f"| ❌ Eval failed for task {t_id}: {e}") |
| 127 | return 0.0 |
| 128 | |
| 129 | tasks = [_safe_eval(res) for res in results] |
| 130 | logger.info(f"| 🚀 Starting batch evaluation for {len(tasks)} items in benchmark '{benchmark_name}'") |