(args)
| 189 | |
| 190 | |
| 191 | def main(args): |
| 192 | model, tokenizer = load_models_tokenizer(args) |
| 193 | |
| 194 | dev_result = {} |
| 195 | for subject_name in tqdm(SUBJECTS): |
| 196 | dev_file_path = os.path.join( |
| 197 | args.eval_data_path, 'dev', f'{subject_name}_dev.csv') |
| 198 | test_file_path = os.path.join( |
| 199 | args.eval_data_path, 'test', f'{subject_name}_test.csv') |
| 200 | |
| 201 | dev_df = pd.read_csv(dev_file_path, names=[ |
| 202 | 'question', 'A', 'B', 'C', 'D', 'answer']) |
| 203 | test_df = pd.read_csv(test_file_path, names=[ |
| 204 | 'question', 'A', 'B', 'C', 'D', 'answer']) |
| 205 | |
| 206 | score = eval_subject(model, tokenizer, subject_name, test_df, dev_df=dev_df, k=5, few_shot=True, |
| 207 | save_result_dir=f"outs/mmlu_eval_result") |
| 208 | dev_result[subject_name] = score |
| 209 | cal_mmlu(dev_result) |
| 210 | |
| 211 | |
| 212 | TASK_NAME_MAPPING = {'stem': ['abstract_algebra', 'anatomy', 'astronomy', 'college_biology', 'college_chemistry', 'college_computer_science', 'college_mathematics', 'college_physics', 'computer_security', 'conceptual_physics', 'electrical_engineering', 'elementary_mathematics', 'high_school_biology', 'high_school_chemistry', 'high_school_computer_science', 'high_school_mathematics', 'high_school_physics', 'high_school_statistics', 'machine_learning'], |
no test coverage detected