| 33 | |
| 34 | |
| 35 | class Asdiv(Task): |
| 36 | VERSION = 0 |
| 37 | DATASET_PATH = inspect.getfile(lm_eval.datasets.asdiv.asdiv) |
| 38 | |
| 39 | def has_training_docs(self): |
| 40 | return False |
| 41 | |
| 42 | def has_validation_docs(self): |
| 43 | return True |
| 44 | |
| 45 | def has_test_docs(self): |
| 46 | return False |
| 47 | |
| 48 | def training_docs(self): |
| 49 | raise NotImplementedError("This dataset has no training docs") |
| 50 | |
| 51 | def validation_docs(self): |
| 52 | return self.dataset["validation"] |
| 53 | |
| 54 | def test_docs(self): |
| 55 | raise NotImplementedError("This dataset has no test docs") |
| 56 | |
| 57 | def fewshot_context( |
| 58 | self, doc, num_fewshot, provide_description=None, rnd=None, description=None |
| 59 | ): |
| 60 | assert num_fewshot == 0, "ASDiv is intended only for the zero-shot setting." |
| 61 | return super().fewshot_context( |
| 62 | doc=doc, num_fewshot=num_fewshot, rnd=rnd, description=description |
| 63 | ) |
| 64 | |
| 65 | def doc_to_text(self, doc): |
| 66 | # TODO: add solution-type |
| 67 | return doc["body"] + "\n" + "Question:" + doc["question"] + "\n" + "Answer:" |
| 68 | |
| 69 | def should_decontaminate(self): |
| 70 | return True |
| 71 | |
| 72 | def doc_to_decontamination_query(self, doc): |
| 73 | return doc["body"] + " " + doc["question"] |
| 74 | |
| 75 | def doc_to_target(self, doc): |
| 76 | # TODO: add formula |
| 77 | |
| 78 | answer = doc["answer"].split(" (")[0] |
| 79 | return " " + answer |
| 80 | |
| 81 | def construct_requests(self, doc, ctx): |
| 82 | ll, is_greedy = rf.loglikelihood(ctx, self.doc_to_target(doc)) |
| 83 | return ll, is_greedy |
| 84 | |
| 85 | def process_results(self, doc, results): |
| 86 | ll, is_greedy = results |
| 87 | |
| 88 | return {"acc": int(is_greedy)} |
| 89 | |
| 90 | def aggregation(self): |
| 91 | return {"acc": mean} |
| 92 |
nothing calls this directly
no outgoing calls
no test coverage detected