(self, request: ExperimentRequest)
| 79 | return task_requests |
| 80 | |
| 81 | def run(self, request: ExperimentRequest) -> ExperimentReport: |
| 82 | results: list[TaskResult] = [] |
| 83 | task_requests = self.build_task_requests(request) |
| 84 | benchmarks = self._dedupe_names(task_request.benchmark.name for task_request in task_requests) |
| 85 | total_rounds = max(request.repeat, 1) |
| 86 | report = ExperimentReport( |
| 87 | benchmark_file=str(request.benchmark_file), |
| 88 | output_file=str(request.output_file) if request.output_file else None, |
| 89 | repeat=total_rounds, |
| 90 | benchmarks=benchmarks, |
| 91 | models=list(request.model_identifiers), |
| 92 | ) |
| 93 | |
| 94 | for round_index in range(total_rounds): |
| 95 | for task_request in task_requests: |
| 96 | result = self.task_runner.run(task_request) |
| 97 | result.details["round_index"] = round_index |
| 98 | results.append(result) |
| 99 | if request.output_file: |
| 100 | write_jsonl_record(request.output_file, result.to_dict()) |
| 101 | report.results.extend(results) |
| 102 | return report |
| 103 | |
| 104 | @staticmethod |
| 105 | def _resolve_benchmarks(benchmark_file: str | Path, benchmark_names: list[str]) -> list[BenchmarkItem]: |
no test coverage detected