Compute LLM alignment statistics from evaluation results.
(evaluated: list[dict])
| 993 | |
| 994 | |
| 995 | def _compute_llm_stats(evaluated: list[dict]) -> dict | None: |
| 996 | """Compute LLM alignment statistics from evaluation results.""" |
| 997 | llm_results = [ |
| 998 | r |
| 999 | for r in evaluated |
| 1000 | if r.get("llm_verdict") and r["llm_verdict"].get("verdict") != "ERROR" |
| 1001 | ] |
| 1002 | if not llm_results: |
| 1003 | return None |
| 1004 | |
| 1005 | verdict_counts = {"ALIGNED": 0, "PARTIALLY_ALIGNED": 0, "NOT_ALIGNED": 0} |
| 1006 | for r in llm_results: |
| 1007 | v = r["llm_verdict"]["verdict"] |
| 1008 | if v in verdict_counts: |
| 1009 | verdict_counts[v] += 1 |
| 1010 | llm_total = len(llm_results) |
| 1011 | return { |
| 1012 | "total_judged": llm_total, |
| 1013 | "aligned": verdict_counts["ALIGNED"], |
| 1014 | "partially_aligned": verdict_counts["PARTIALLY_ALIGNED"], |
| 1015 | "not_aligned": verdict_counts["NOT_ALIGNED"], |
| 1016 | "alignment_rate": verdict_counts["ALIGNED"] / llm_total, |
| 1017 | "partial_or_better_rate": ( |
| 1018 | verdict_counts["ALIGNED"] + verdict_counts["PARTIALLY_ALIGNED"] |
| 1019 | ) |
| 1020 | / llm_total, |
| 1021 | } |
| 1022 | |
| 1023 | |
| 1024 | def compute_aggregate_stats( |
no test coverage detected