(model, max_seq_len, data_path, ntrain=-1, few_shot = True)
| 150 | return prompt |
| 151 | |
| 152 | def run_infer_eval(model, max_seq_len, data_path, ntrain=-1, few_shot = True): |
| 153 | |
| 154 | total_results = {} |
| 155 | for task in subcategories.keys(): |
| 156 | |
| 157 | print('Testing %s ...' % task) |
| 158 | test_file_path = os.path.join(data_path, "test", f"{task}.csv") |
| 159 | test_df = pd.read_csv(test_file_path) |
| 160 | dev_file_path = os.path.join(data_path, "dev", f"{task}.csv") |
| 161 | dev_df = pd.read_csv(dev_file_path) |
| 162 | few_shot_prompt = generate_few_shot_prompt(task, dev_df, ntrain) if few_shot else [] |
| 163 | |
| 164 | results = [] |
| 165 | for _, row in tqdm(test_df.iterrows(), total=len(test_df)): |
| 166 | prompt = format_example(row, include_answer=False) |
| 167 | full_prompt = resize_prompt( |
| 168 | model.tokenizer, |
| 169 | max_seq_len, |
| 170 | few_shot_prompt+prompt |
| 171 | ) |
| 172 | output = model.generate( |
| 173 | prompts=[full_prompt], |
| 174 | images=None, |
| 175 | max_gen_len=100, |
| 176 | return_logits=True |
| 177 | ) |
| 178 | pred = extract_ans_by_logits(tokenizer = model.tokenizer, logits=output) |
| 179 | |
| 180 | results.append(pred == row['Answer']) |
| 181 | total_results[task] = sum(results) / len(results) |
| 182 | |
| 183 | return total_results |
| 184 | |
| 185 | def cal_cmmlu(res): |
| 186 | print("\n\n\n") |
no test coverage detected