Evaluate multiple gold/model answer pairs and aggregate metrics.
(
pairs: list[tuple[Any, Any]],
*,
similarity_threshold: float = 0.0,
)
| 393 | ) |
| 394 | |
| 395 | def evaluate_static_json_batch( |
| 396 | pairs: list[tuple[Any, Any]], |
| 397 | *, |
| 398 | similarity_threshold: float = 0.0, |
| 399 | ) -> dict[str, Any]: |
| 400 | """Evaluate multiple gold/model answer pairs and aggregate metrics.""" |
| 401 | scores = [ |
| 402 | evaluate_static_json( |
| 403 | gold, |
| 404 | model, |
| 405 | similarity_threshold=similarity_threshold, |
| 406 | ) |
| 407 | for gold, model in pairs |
| 408 | ] |
| 409 | |
| 410 | if not scores: |
| 411 | return { |
| 412 | "num_examples": 0, |
| 413 | "partial_exact_match_accuracy": 0.0, |
| 414 | "strict_exact_match_accuracy": 0.0, |
| 415 | "partial_similarity_score": 0.0, |
| 416 | "precision": 0.0, |
| 417 | "recall": 0.0, |
| 418 | "f1": 0.0, |
| 419 | "examples": [], |
| 420 | } |
| 421 | |
| 422 | return { |
| 423 | "num_examples": len(scores), |
| 424 | "partial_exact_match_accuracy": sum( |
| 425 | score.partial_exact_match_accuracy for score in scores |
| 426 | ) |
| 427 | / len(scores), |
| 428 | "strict_exact_match_accuracy": sum( |
| 429 | score.strict_exact_match_accuracy for score in scores |
| 430 | ) |
| 431 | / len(scores), |
| 432 | "partial_similarity_score": sum( |
| 433 | score.partial_similarity_score for score in scores |
| 434 | ) |
| 435 | / len(scores), |
| 436 | "precision": sum(score.precision for score in scores) / len(scores), |
| 437 | "recall": sum(score.recall for score in scores) / len(scores), |
| 438 | "f1": sum(score.f1 for score in scores) / len(scores), |
| 439 | "examples": [score.to_dict() for score in scores], |
| 440 | } |
| 441 | |
| 442 | class StaticJsonScorer: |
| 443 | """Evaluation scorer wrapper for the trajectory-based pipeline.""" |