(args)
| 213 | return score, total_results, invalid_outputs |
| 214 | |
| 215 | def main(args): |
| 216 | |
| 217 | path_split = args.pretrained_path.split('/') |
| 218 | if path_split[-1] == '': |
| 219 | path_split.pop(-1) |
| 220 | model_name = path_split[-1] |
| 221 | infer_path = os.path.join('results', model_name, 'mmlu/infer') |
| 222 | os.makedirs(infer_path, exist_ok=True) |
| 223 | eval_path = os.path.join('results', model_name, 'mmlu/eval') |
| 224 | os.makedirs(eval_path, exist_ok=True) |
| 225 | |
| 226 | model = load(args) |
| 227 | |
| 228 | run_infer(model, args.max_seq_len, TASKS, infer_path, args.ntrain, args.overwrite) |
| 229 | |
| 230 | torch.distributed.barrier() |
| 231 | if torch.distributed.get_rank() == 0: |
| 232 | |
| 233 | score, _ , invalid_outputs= run_eval(TASKS, infer_path) |
| 234 | |
| 235 | with open(os.path.join(eval_path, 'run_results.json'), 'w') as f: |
| 236 | json.dump(score, f, ensure_ascii=False, indent=2) |
| 237 | |
| 238 | with open(os.path.join(eval_path, 'debug_invalid_outputs.jsonl'), 'w') as outfile: |
| 239 | for entry in invalid_outputs: |
| 240 | json.dump(entry, outfile, ensure_ascii=False,indent=2) |
| 241 | outfile.write('\n') |
| 242 | |
| 243 | |
| 244 | if __name__ == "__main__": |
no test coverage detected