| 191 | writer.write(record) |
| 192 | |
| 193 | def run_eval(infer_path): |
| 194 | |
| 195 | infer_file = os.path.join(infer_path, 'gsm8k_infer.jsonl') |
| 196 | assert os.path.exists(infer_file) , f'ERROR: please run inference first!' |
| 197 | |
| 198 | score = {} |
| 199 | results = [] |
| 200 | invalid_outputs = [] |
| 201 | with jsonlines.open(infer_file) as f: |
| 202 | for item in f.iter(type=dict, skip_invalid=True): |
| 203 | pred = extract_ans(item['completion']) |
| 204 | if pred != None: |
| 205 | results.append(float(pred) == float(item['target_ans'])) |
| 206 | else: |
| 207 | results.append(False) |
| 208 | temp = { |
| 209 | 'output_split': re.split('Quetion:', item['completion'], flags=re.IGNORECASE)[0], |
| 210 | 'answer': item['target_ans'] |
| 211 | } |
| 212 | invalid_outputs.append(temp) |
| 213 | |
| 214 | score['TOTAL_AVERAGE'] = '%.4f' %(sum(results) / len(results)) |
| 215 | |
| 216 | return score, invalid_outputs |
| 217 | |
| 218 | def main(args): |
| 219 | |