(self, predictions, references, test_set)
| 92 | self.circular_pattern = circular_pattern |
| 93 | |
| 94 | def score(self, predictions, references, test_set): |
| 95 | from opencompass.datasets.circular import (get_all_possible_patterns, |
| 96 | get_circular_patterns, |
| 97 | get_origin_patterns) |
| 98 | |
| 99 | circular_patterns = {} |
| 100 | circular_patterns['origin'] = get_origin_patterns( |
| 101 | test_set[0]['circular_pattern']) |
| 102 | circular_patterns['circular'] = get_circular_patterns( |
| 103 | test_set[0]['circular_pattern']) |
| 104 | if self.circular_pattern == 'all_possible': |
| 105 | circular_patterns['all_possible'] = get_all_possible_patterns( |
| 106 | test_set[0]['circular_pattern']) |
| 107 | |
| 108 | metrics = {} |
| 109 | tmp_metrics = {} |
| 110 | tmp_metrics.update({f'correct_{k}': 0 for k in circular_patterns}) |
| 111 | tmp_metrics.update({f'count_{k}': 0 for k in circular_patterns}) |
| 112 | # calculate the original accuracy |
| 113 | for pred, refr, origin_item in zip(predictions, references, test_set): |
| 114 | parsed = self.match_any_label(pred, origin_item) |
| 115 | circular_pattern = origin_item['circular_pattern'] |
| 116 | for k in circular_patterns: |
| 117 | if tuple(circular_pattern) in circular_patterns[k]: |
| 118 | tmp_metrics[f'correct_{k}'] += 1 if parsed == refr else 0 |
| 119 | tmp_metrics[f'count_{k}'] += 1 |
| 120 | |
| 121 | for k in circular_patterns: |
| 122 | metrics[f'acc_{k}'] = (tmp_metrics[f'correct_{k}'] / |
| 123 | tmp_metrics[f'count_{k}'] * 100) |
| 124 | |
| 125 | # calculate the circular accuracy |
| 126 | _details = {k: {} for k in circular_patterns} |
| 127 | for pred, refr, origin_item in zip(predictions, references, test_set): |
| 128 | index = origin_item['qid'] |
| 129 | parsed = self.match_any_label(pred, origin_item) |
| 130 | circular_pattern = origin_item['circular_pattern'] |
| 131 | for k in circular_patterns: |
| 132 | if tuple(circular_pattern) in circular_patterns[k]: |
| 133 | _details[k].setdefault( |
| 134 | index, []).append(True if parsed == refr else False) |
| 135 | for k in _details: |
| 136 | _details[k] = { |
| 137 | index: sum(_details[k][index]) |
| 138 | for index in _details[k] |
| 139 | } |
| 140 | for k in _details: |
| 141 | for j in range(1, len(circular_patterns[k]) + 1): |
| 142 | count = sum([_details[k][index] >= j for index in _details[k]]) |
| 143 | total = len(_details[k]) |
| 144 | if j != len(circular_patterns[k]): |
| 145 | metrics[f'more_{j}_{k}'] = count / total * 100 |
| 146 | else: |
| 147 | metrics[f'perf_{k}'] = count / total * 100 |
| 148 | |
| 149 | # make details |
| 150 | details = {} |
| 151 | for index in range(len(predictions)): |
nothing calls this directly
no test coverage detected