| 175 | return total_results |
| 176 | |
| 177 | def cal_ceval(res): |
| 178 | results = {} |
| 179 | acc_sum_dict = dict() |
| 180 | acc_norm_sum_dict = dict() |
| 181 | cnt_dict = dict() |
| 182 | acc_sum = 0.0 |
| 183 | cnt = 0 |
| 184 | hard_cnt = 0 |
| 185 | hard_acc_sum = 0.0 |
| 186 | for tt in res.keys(): |
| 187 | name = tt.split("-")[-1] |
| 188 | acc_sum += float(res[tt]) |
| 189 | cnt += 1 |
| 190 | class_ = TASK_NAME_MAPPING[name][2] |
| 191 | if class_ not in acc_sum_dict: |
| 192 | acc_sum_dict[class_] = 0.0 |
| 193 | acc_norm_sum_dict[class_] = 0.0 |
| 194 | cnt_dict[class_] = 0.0 |
| 195 | if name in hard_list: |
| 196 | hard_cnt += 1 |
| 197 | hard_acc_sum += float(res[tt]) |
| 198 | acc_sum_dict[class_] += float(res[tt]) |
| 199 | cnt_dict[class_] += 1 |
| 200 | print("\n\n\n") |
| 201 | for k in ["STEM", "Social Science", "Humanities", "Other"]: |
| 202 | if k in cnt_dict: |
| 203 | results[k] = "%.4f" %(acc_sum_dict[k] / cnt_dict[k]) |
| 204 | print("%s acc: %.4f " % (k, acc_sum_dict[k] / cnt_dict[k])) |
| 205 | if hard_cnt > 0: |
| 206 | results['Hard'] = "%.4f" %(hard_acc_sum / hard_cnt) |
| 207 | print("Hard acc:%.4f " % (hard_acc_sum / hard_cnt)) |
| 208 | results ['AVERAGE'] = "%.4f" % (acc_sum / cnt) |
| 209 | print("AVERAGE acc:%.4f " % (acc_sum / cnt)) |
| 210 | return results |
| 211 | |
| 212 | def main(args): |
| 213 | |