(
model,
tokenizer,
subject_name,
test_df,
k=5,
dev_df=None,
few_shot=False,
save_result_dir=None,
**kwargs,
)
| 126 | |
| 127 | @torch.no_grad() |
| 128 | def eval_subject( |
| 129 | model, |
| 130 | tokenizer, |
| 131 | subject_name, |
| 132 | test_df, |
| 133 | k=5, |
| 134 | dev_df=None, |
| 135 | few_shot=False, |
| 136 | save_result_dir=None, |
| 137 | **kwargs, |
| 138 | ): |
| 139 | file_path = os.path.join(save_result_dir, f"{subject_name}_result.csv") if save_result_dir else None |
| 140 | if file_path and os.path.exists(file_path): |
| 141 | # Read the file, extract the 'correctness' column, and calculate correct_ratio |
| 142 | existing_df = pd.read_csv(file_path, encoding="utf-8") |
| 143 | if "correctness" in existing_df: |
| 144 | correct_ratio = 100 * existing_df["correctness"].sum() / len(existing_df["correctness"]) |
| 145 | return correct_ratio |
| 146 | |
| 147 | result = [] |
| 148 | score = [] |
| 149 | |
| 150 | few_shot_prompt = ( |
| 151 | generate_few_shot_prompt(k, subject_name, dev_df) if few_shot else "" |
| 152 | ) |
| 153 | all_probs = {"prob_A": [], "prob_B": [], "prob_C": [], "prob_D": []} |
| 154 | if args.debug: |
| 155 | print(f"few_shot_prompt: {few_shot_prompt}") |
| 156 | |
| 157 | for _, row in tqdm(test_df.iterrows(), total=len(test_df)): |
| 158 | question = format_example(row, subject_name, include_answer=False) |
| 159 | full_prompt = few_shot_prompt + question |
| 160 | |
| 161 | output, input_info = get_logits(tokenizer, model, [full_prompt]) |
| 162 | assert output.shape[0] == 1 |
| 163 | logits = output.flatten() |
| 164 | |
| 165 | softval = torch.nn.functional.softmax( |
| 166 | torch.tensor( |
| 167 | [ |
| 168 | logits[tokenizer("A")["input_ids"][-1]], |
| 169 | logits[tokenizer("B")["input_ids"][-1]], |
| 170 | logits[tokenizer("C")["input_ids"][-1]], |
| 171 | logits[tokenizer("D")["input_ids"][-1]], |
| 172 | ] |
| 173 | ), |
| 174 | dim=0, |
| 175 | ) |
| 176 | if softval.dtype in {torch.bfloat16, torch.float16}: |
| 177 | softval = softval.to(dtype=torch.float32) |
| 178 | probs = softval.detach().cpu().numpy() |
| 179 | |
| 180 | for i, choice in enumerate(choices): |
| 181 | all_probs[f"prob_{choice}"].append(probs[i]) |
| 182 | pred = {0: "A", 1: "B", 2: "C", 3: "D"}[np.argmax(probs)] |
| 183 | |
| 184 | if "answer" in row: |
| 185 | correct = 1 if pred == row["answer"] else 0 |
no test coverage detected