| 172 | writer.write(record) |
| 173 | |
| 174 | def run_eval(infer_path): |
| 175 | |
| 176 | score = {} |
| 177 | infer_file = os.path.join(infer_path, 'math_infer.jsonl') |
| 178 | assert os.path.exists(infer_file) , f'ERROR: please run inference first!' |
| 179 | |
| 180 | results = [] |
| 181 | invalid_outputs = [] |
| 182 | with jsonlines.open(infer_file) as f: |
| 183 | for item in f.iter(type=dict, skip_invalid=True): |
| 184 | pred = extract_ans(item['completion'], item['target_ans']) |
| 185 | if pred != None: |
| 186 | results.append(math_util.is_equiv(pred, item['target_ans'])) |
| 187 | else: |
| 188 | results.append(False) |
| 189 | temp = { |
| 190 | 'output_split': re.split("Problem:", item['completion'], flags=re.IGNORECASE)[0], |
| 191 | 'answer':item['target_ans'] |
| 192 | } |
| 193 | invalid_outputs.append(temp) |
| 194 | |
| 195 | score['TOTAL_AVERAGE'] = '%.4f' %(sum(results) / len(results)) |
| 196 | |
| 197 | return score, invalid_outputs |
| 198 | |
| 199 | def main(args): |
| 200 | |