(
model,
tokenizer,
subject_name,
test_df,
k=5,
dev_df=None,
few_shot=False,
save_result_dir=None,
**kwargs,
)
| 83 | |
| 84 | @torch.no_grad() |
| 85 | def eval_subject( |
| 86 | model, |
| 87 | tokenizer, |
| 88 | subject_name, |
| 89 | test_df, |
| 90 | k=5, |
| 91 | dev_df=None, |
| 92 | few_shot=False, |
| 93 | save_result_dir=None, |
| 94 | **kwargs, |
| 95 | ): |
| 96 | file_path = os.path.join(save_result_dir, f"{subject_name}_result.csv") if save_result_dir else None |
| 97 | if file_path and os.path.exists(file_path): |
| 98 | # Read the file, extract the 'correctness' column, and calculate correct_ratio |
| 99 | existing_df = pd.read_csv(file_path, encoding="utf-8") |
| 100 | if "correctness" in existing_df: |
| 101 | return list(existing_df["correctness"]) |
| 102 | result = [] |
| 103 | score = [] |
| 104 | |
| 105 | few_shot_prompt = ( |
| 106 | generate_few_shot_prompt(k, subject_name, dev_df) if few_shot else [] |
| 107 | ) |
| 108 | all_probs = {"prob_A": [], "prob_B": [], "prob_C": [], "prob_D": []} |
| 109 | if args.debug: |
| 110 | print(f"few_shot_prompt: {few_shot_prompt}") |
| 111 | |
| 112 | for _, row in tqdm(test_df.iterrows(), total=len(test_df)): |
| 113 | question = format_example(row, include_answer=False) |
| 114 | full_prompt = few_shot_prompt + question |
| 115 | |
| 116 | output, input_info = get_logits(tokenizer, model, [full_prompt]) |
| 117 | assert output.shape[0] == 1 |
| 118 | logits = output.flatten() |
| 119 | |
| 120 | softval = torch.nn.functional.softmax( |
| 121 | torch.tensor( |
| 122 | [ |
| 123 | logits[tokenizer("A")["input_ids"][-1]], |
| 124 | logits[tokenizer("B")["input_ids"][-1]], |
| 125 | logits[tokenizer("C")["input_ids"][-1]], |
| 126 | logits[tokenizer("D")["input_ids"][-1]], |
| 127 | ] |
| 128 | ), |
| 129 | dim=0, |
| 130 | ) |
| 131 | if softval.dtype in {torch.bfloat16, torch.float16}: |
| 132 | softval = softval.to(dtype=torch.float32) |
| 133 | probs = softval.detach().cpu().numpy() |
| 134 | |
| 135 | for i, choice in enumerate(choices): |
| 136 | all_probs[f"prob_{choice}"].append(probs[i]) |
| 137 | pred = {0: "A", 1: "B", 2: "C", 3: "D"}[np.argmax(probs)] |
| 138 | |
| 139 | if "answer" in row: |
| 140 | correct = 1 if pred == row["answer"] else 0 |
| 141 | score.append(correct) |
| 142 | if args.debug: |
no test coverage detected