| 168 | |
| 169 | @dataclass |
| 170 | class BenchmarkConfig: |
| 171 | valid_datasets: list[str] |
| 172 | load_dataset: Callable[[str], Any] |
| 173 | image_name: Callable[[str], str] |
| 174 | problem_statement: Callable[[dict, Path], Any] |
| 175 | working_dir: Callable[[str], str] |
| 176 | evaluate_harness: Callable[..., list[str]] |
| 177 | evaluate_harness_before: Callable[..., Any] |
| 178 | evaluate_harness_after: Callable[..., Any] |
| 179 | |
| 180 | |
| 181 | BENCHMARK_CONFIG: dict[str, BenchmarkConfig] = { |