Print and return per-key-point accuracy plus the all-points-correct ratio.
(results: list)
| 200 | |
| 201 | |
| 202 | def report(results: list) -> dict: |
| 203 | """Print and return per-key-point accuracy plus the all-points-correct ratio.""" |
| 204 | benchmark: dict = {} |
| 205 | all_correct = 0 |
| 206 | |
| 207 | for item in results: |
| 208 | scores = item["res_score_list"] |
| 209 | if scores and 0 not in scores: |
| 210 | all_correct += 1 |
| 211 | for key_point, score in zip(item["prompt_points"], scores): |
| 212 | stats = benchmark.setdefault(key_point, {"all": 0, "right": 0}) |
| 213 | stats["all"] += 1 |
| 214 | stats["right"] += score |
| 215 | |
| 216 | total = len(results) or 1 |
| 217 | logger.info( |
| 218 | "All key points correct: %d / %d (%.4f)", |
| 219 | all_correct, |
| 220 | len(results), |
| 221 | all_correct / total, |
| 222 | ) |
| 223 | for key_point, stats in benchmark.items(): |
| 224 | logger.info("%s: %.4f", key_point, stats["right"] / stats["all"]) |
| 225 | |
| 226 | return { |
| 227 | "all_correct": all_correct, |
| 228 | "total": len(results), |
| 229 | "all_correct_ratio": all_correct / total, |
| 230 | "per_key_point": { |
| 231 | k: v["right"] / v["all"] for k, v in benchmark.items() |
| 232 | }, |
| 233 | } |
| 234 | |
| 235 | |
| 236 | def parse_args() -> argparse.Namespace: |