(
benchmark_file: str,
benchmark_names: list[str] | None,
model_identifiers: list[str],
ip_addr: str,
repeat: int,
prompt_bundle_name: str = "default",
config_file: str | None = None,
output_file: str | None = None,
)
| 436 | |
| 437 | |
| 438 | def handle_experiment( |
| 439 | benchmark_file: str, |
| 440 | benchmark_names: list[str] | None, |
| 441 | model_identifiers: list[str], |
| 442 | ip_addr: str, |
| 443 | repeat: int, |
| 444 | prompt_bundle_name: str = "default", |
| 445 | config_file: str | None = None, |
| 446 | output_file: str | None = None, |
| 447 | ) -> int: |
| 448 | config = _resolve_config(config_file) |
| 449 | resolved_benchmark_file = _resolve_benchmark_file(benchmark_file) |
| 450 | resolved_output_file = Path(output_file) if output_file else _default_experiment_output_path() |
| 451 | |
| 452 | request = ExperimentRequest( |
| 453 | benchmark_file=resolved_benchmark_file, |
| 454 | benchmark_names=benchmark_names or [], |
| 455 | model_identifiers=model_identifiers, |
| 456 | ip_addr=ip_addr, |
| 457 | repeat=repeat, |
| 458 | config=config, |
| 459 | output_file=resolved_output_file, |
| 460 | prompt_bundle_name=prompt_bundle_name, |
| 461 | ) |
| 462 | runner = ExperimentRunner() |
| 463 | report = runner.run(request) |
| 464 | payload = report.to_dict() |
| 465 | print(json.dumps(payload, ensure_ascii=False, indent=2)) |
| 466 | return 0 if payload["summary"].get("error", 0) == 0 else 1 |
| 467 | |
| 468 | |
| 469 | def main(argv: Sequence[str] | None = None) -> int: |
no test coverage detected