Parameters ---------- predicted : dict {question: {'answer': , 'reference': ...}, ...} ground_truth : dict {question: ' . full answer', ...} aspects : dict {question: ' ', ...} Returns ------- overall_accuracy : flo
(predicted, ground_truth, aspects)
| 868 | |
| 869 | |
| 870 | def compute_accuracy(predicted, ground_truth, aspects): |
| 871 | """ |
| 872 | Parameters |
| 873 | ---------- |
| 874 | predicted : dict |
| 875 | {question: {'answer': <letter>, 'reference': ...}, ...} |
| 876 | ground_truth : dict |
| 877 | {question: '<letter>. full answer', ...} |
| 878 | aspects : dict |
| 879 | {question: '<aspect name>', ...} |
| 880 | |
| 881 | Returns |
| 882 | ------- |
| 883 | overall_accuracy : float |
| 884 | aspect_summary : dict |
| 885 | { |
| 886 | '<aspect name>': { |
| 887 | 'total': <int>, # questions in this aspect |
| 888 | 'correct': <int>, # correctly answered questions |
| 889 | 'accuracy': <float> # correct / total (0–1) |
| 890 | }, |
| 891 | ... |
| 892 | } |
| 893 | """ |
| 894 | correct_global = 0 |
| 895 | total_global = len(ground_truth) |
| 896 | |
| 897 | total_by_aspect = defaultdict(int) |
| 898 | correct_by_aspect = defaultdict(int) |
| 899 | |
| 900 | for q, pred_info in predicted.items(): |
| 901 | letter_pred = pred_info['answer'] |
| 902 | ref = pred_info.get('reference', 'NA') |
| 903 | |
| 904 | # Count this question toward its aspect, even if NA or missing gt |
| 905 | aspect = aspects.get(q, 'Unknown') |
| 906 | total_by_aspect[aspect] += 1 |
| 907 | |
| 908 | if letter_pred == 'NA' or ref == 'NA': |
| 909 | continue # automatically wrong |
| 910 | |
| 911 | if q in ground_truth: |
| 912 | letter_gt = ground_truth[q].split('.')[0].strip() |
| 913 | |
| 914 | if len(letter_pred) > 0: |
| 915 | letter_pred = letter_pred[0].upper() |
| 916 | if letter_pred == letter_gt: |
| 917 | correct_global += 1 |
| 918 | correct_by_aspect[aspect] += 1 |
| 919 | |
| 920 | overall_accuracy = correct_global / total_global if total_global else 0.0 |
| 921 | |
| 922 | # Build the per-aspect dictionary |
| 923 | aspect_summary = {} |
| 924 | for aspect, total in total_by_aspect.items(): |
| 925 | correct = correct_by_aspect[aspect] |
| 926 | acc = correct / total if total else 0.0 |
| 927 | aspect_summary[aspect] = { |
no outgoing calls
no test coverage detected