| 187 | writer.write(record) |
| 188 | |
| 189 | def run_eval(tasks, infer_path, mode): |
| 190 | |
| 191 | score = {} |
| 192 | total_results=[] |
| 193 | for task in tasks: |
| 194 | |
| 195 | task_infer_path = os.path.join(infer_path, f'{task}_infer.jsonl') |
| 196 | assert os.path.exists(task_infer_path) , f'ERROR: please run {task} inference first!' |
| 197 | |
| 198 | results = [] |
| 199 | with jsonlines.open(task_infer_path) as f: |
| 200 | for item in f.iter(type=dict, skip_invalid=True): |
| 201 | pred = extract_ans(item['completion'], mode) |
| 202 | results.append(pred == item['target_ans']) |
| 203 | score[task] = '%.4f' %(sum(results) / len(results)) |
| 204 | total_results.extend(results) |
| 205 | score['TOTAL_AVERAGE'] = '%.4f' %(sum(total_results) / len(total_results)) |
| 206 | |
| 207 | return score, total_results |
| 208 | |
| 209 | |
| 210 | def main(args, multiple_choice_tasks=MULTIPLE_CHOICE_TASKS, free_form_tasks=FREE_FORM_TASKS): |