()
| 266 | |
| 267 | |
| 268 | def main() -> None: |
| 269 | args = parse_args() |
| 270 | |
| 271 | judge = GeminiJudge( |
| 272 | model=args.model, api_key=args.api_key, max_retries=args.max_retries |
| 273 | ) |
| 274 | |
| 275 | dataset = load_dataset(DATASET_NAME)[args.split] |
| 276 | if args.num_samples is not None: |
| 277 | dataset = dataset.select(range(min(args.num_samples, len(dataset)))) |
| 278 | |
| 279 | results = [eval_sample(sample, judge) for sample in tqdm(dataset, desc="Evaluating")] |
| 280 | |
| 281 | write_json(results, args.output) |
| 282 | logger.info("Saved per-sample results to %s", args.output) |
| 283 | report(results) |
| 284 | |
| 285 | |
| 286 | if __name__ == "__main__": |
no test coverage detected