(self, doc, ctx)
| 253 | return f"{answer}\nIs the answer correct? {label_str}" |
| 254 | |
| 255 | def construct_requests(self, doc, ctx): |
| 256 | true_choice = self.format_answer(answer=doc["answer"], label=True) |
| 257 | false_choice = self.format_answer(answer=doc["answer"], label=False) |
| 258 | |
| 259 | ll_true_choice, _ = rf.loglikelihood(ctx, f" {true_choice}") |
| 260 | ll_false_choice, _ = rf.loglikelihood(ctx, f" {false_choice}") |
| 261 | |
| 262 | return ll_true_choice, ll_false_choice |
| 263 | |
| 264 | def process_results(self, doc, results): |
| 265 | ll_true_choice, ll_false_choice = results |
nothing calls this directly
no test coverage detected