(args)
| 197 | return results |
| 198 | |
| 199 | def main(args): |
| 200 | |
| 201 | path_split = args.pretrained_path.split('/') |
| 202 | if path_split[-1] == '': |
| 203 | path_split.pop(-1) |
| 204 | model_name = path_split[-1] |
| 205 | eval_path = os.path.join('results', model_name, 'cmmlu/eval') |
| 206 | os.makedirs(eval_path, exist_ok=True) |
| 207 | |
| 208 | model = load(args) |
| 209 | |
| 210 | result_path = os.path.join(eval_path, 'run_results.json') |
| 211 | if not args.overwrite and os.path.exists(result_path): |
| 212 | print(f"{result_path} existed, skip!") |
| 213 | return |
| 214 | |
| 215 | subjects_result = run_infer_eval(model, args.max_seq_len, args.data_dir, args.ntrain) |
| 216 | |
| 217 | torch.distributed.barrier() |
| 218 | if torch.distributed.get_rank() == 0: |
| 219 | |
| 220 | score = cal_cmmlu(subjects_result) |
| 221 | with open(result_path, 'w') as f: |
| 222 | json.dump(score, f, ensure_ascii=False, indent=2) |
| 223 | |
| 224 | if __name__ == "__main__": |
| 225 |
no test coverage detected