| 185 | writer.close() |
| 186 | |
| 187 | def run_eval(tasks, infer_path): |
| 188 | |
| 189 | score = {} |
| 190 | total_results=[] |
| 191 | invalid_outputs = [] |
| 192 | for task in tasks: |
| 193 | |
| 194 | task_infer_path = os.path.join(infer_path, f'{task}_infer.jsonl') |
| 195 | assert os.path.exists(task_infer_path) , f'ERROR: please run {task} inference first!' |
| 196 | |
| 197 | results = [] |
| 198 | invalid = [] |
| 199 | with jsonlines.open(task_infer_path) as f: |
| 200 | for item in f.iter(type=dict, skip_invalid=True): |
| 201 | pred = extract_ans(item['completion']) |
| 202 | if pred == None: |
| 203 | invalid.append( |
| 204 | {'output': item['completion'], |
| 205 | 'answer': item['target_ans']} |
| 206 | ) |
| 207 | results.append(pred == item['target_ans']) |
| 208 | score[task] = '%.4f' %(sum(results) / len(results)) |
| 209 | total_results.extend(results) |
| 210 | invalid_outputs.extend(invalid) |
| 211 | score['TOTAL_AVERAGE'] = '%.4f' %(sum(total_results) / len(total_results)) |
| 212 | |
| 213 | return score, total_results, invalid_outputs |
| 214 | |
| 215 | def main(args): |
| 216 | |