(items)
| 65 | |
| 66 | |
| 67 | def acc_all_stderr(items): |
| 68 | # Only count as correct if all answers are labeled correctly for each question |
| 69 | question_scoring_dict = {} |
| 70 | preds = list(zip(*items))[0] |
| 71 | docs = list(zip(*items))[1] |
| 72 | |
| 73 | for doc, pred in zip(docs, preds): |
| 74 | question_id = doc["idx"]["question"] |
| 75 | if question_id not in question_scoring_dict: |
| 76 | question_scoring_dict[question_id] = [] |
| 77 | |
| 78 | gold_label = doc["label"] == 1 |
| 79 | question_scoring_dict[question_id].append(gold_label == pred) |
| 80 | |
| 81 | acc = mean_stderr([int(all(x)) for x in question_scoring_dict.values()]) |
| 82 | return acc |
| 83 | |
| 84 | |
| 85 | def metric_max_over_ground_truths(metric_fn, prediction, ground_truths): |
nothing calls this directly
no test coverage detected