| 154 | |
| 155 | |
| 156 | def cal_mmlu(res): |
| 157 | acc_sum_dict = dict() |
| 158 | acc_norm_sum_dict = dict() |
| 159 | cnt_dict = dict() |
| 160 | acc_sum = 0. |
| 161 | cnt = 0 |
| 162 | hard_cnt = 0 |
| 163 | hard_acc_sum = 0. |
| 164 | |
| 165 | for class_ in TASK_NAME_MAPPING.keys(): |
| 166 | acc_sum_dict[class_] = 0. |
| 167 | acc_norm_sum_dict[class_] = 0. |
| 168 | cnt_dict[class_] = 0. |
| 169 | |
| 170 | for tt in TASK_NAME_MAPPING[class_]: |
| 171 | acc_sum += sum(res[tt]) |
| 172 | cnt += len(res[tt]) |
| 173 | |
| 174 | acc_sum_dict[class_] += sum(res[tt]) |
| 175 | cnt_dict[class_] += len(res[tt]) |
| 176 | |
| 177 | for k in TASK_NAME_MAPPING.keys(): |
| 178 | if k in cnt_dict: |
| 179 | print('%s ACC: %.2f ' % ( |
| 180 | k, acc_sum_dict[k] / cnt_dict[k] * 100)) |
| 181 | timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") |
| 182 | with open(f"mmlu_eval_result_{timestamp}.json", "w") as f: |
| 183 | result = {} |
| 184 | result["acc"] = acc_sum / cnt * 100 |
| 185 | result["cnt"] = cnt |
| 186 | result["acc_sum_dict"] = acc_sum_dict |
| 187 | result["cnt_dict"] = cnt_dict |
| 188 | f.write(json.dumps(result)) |
| 189 | |
| 190 | |
| 191 | def main(args): |