Save benchmark results to JSON file with real-time updates.
| 34 | TARGET_MODEL = "openrouter/gemini-3-flash-preview" |
| 35 | |
| 36 | class BenchmarkResultSaver: |
| 37 | """Save benchmark results to JSON file with real-time updates.""" |
| 38 | |
| 39 | def __init__(self, benchmark_name: str, concurrency: int, total_tasks: int, model_name: str): |
| 40 | self.benchmark_name = benchmark_name |
| 41 | self.concurrency = concurrency |
| 42 | self.total_tasks = total_tasks |
| 43 | self.model_name = model_name |
| 44 | self.start_time = datetime.now() |
| 45 | |
| 46 | # Create results directory if it doesn't exist |
| 47 | self.results_dir = Path(__file__).parent / "workdir/results" |
| 48 | self.results_dir.mkdir(parents=True, exist_ok=True) |
| 49 | |
| 50 | # Generate filename with timestamp |
| 51 | timestamp = self.start_time.strftime("%Y-%m-%d_%H-%M-%S") |
| 52 | self.filename = f"benchmark_{benchmark_name}_{timestamp}.json" |
| 53 | self.filepath = self.results_dir / self.filename |
| 54 | |
| 55 | # Initialize thread lock for file operations |
| 56 | self.file_lock = asyncio.Lock() |
| 57 | |
| 58 | # Initialize results structure |
| 59 | self.results_data = { |
| 60 | "experiment_meta": { |
| 61 | "timestamp": self.start_time.isoformat() + "Z", |
| 62 | "benchmark": benchmark_name, |
| 63 | "concurrency": concurrency, |
| 64 | "total_tasks": total_tasks, |
| 65 | "model": model_name |
| 66 | }, |
| 67 | "results": [], |
| 68 | "summary": { |
| 69 | "completed_tasks": 0, |
| 70 | "correct_answers": 0, |
| 71 | "accuracy": 0.0, |
| 72 | "last_updated": self.start_time.isoformat() + "Z" |
| 73 | } |
| 74 | } |
| 75 | |
| 76 | # Save initial empty results |
| 77 | asyncio.create_task(self._save_to_file()) |
| 78 | |
| 79 | async def add_task_result(self, task: Task, processing_time: float = None): |
| 80 | """Add a single task result and update the file.""" |
| 81 | async with self.file_lock: |
| 82 | task_result = { |
| 83 | "task_id": task.task_id, |
| 84 | "task_input": task.input[:200] + "..." if len(task.input) > 200 else task.input, |
| 85 | "ground_truth": str(task.ground_truth) if task.ground_truth else "", |
| 86 | "result": str(task.result) if task.result else "", |
| 87 | "reasoning": getattr(task, 'reasoning', ""), |
| 88 | "correct": task.score == 1.0 if task.score is not None else False, |
| 89 | "processing_time": processing_time or getattr(task, 'time', 0.0) |
| 90 | } |
| 91 | |
| 92 | self.results_data["results"].append(task_result) |
| 93 |