| 27 | |
| 28 | |
| 29 | class Arithmetic(Task): |
| 30 | VERSION = 0 |
| 31 | DATASET_PATH = "EleutherAI/arithmetic" |
| 32 | |
| 33 | def has_training_docs(self): |
| 34 | return False |
| 35 | |
| 36 | def has_validation_docs(self): |
| 37 | return True |
| 38 | |
| 39 | def has_test_docs(self): |
| 40 | return False |
| 41 | |
| 42 | def training_docs(self): |
| 43 | return NotImplemented |
| 44 | |
| 45 | def validation_docs(self): |
| 46 | return self.dataset["validation"] |
| 47 | |
| 48 | def test_docs(self): |
| 49 | return NotImplemented |
| 50 | |
| 51 | def doc_to_text(self, doc): |
| 52 | return doc["context"] |
| 53 | |
| 54 | def should_decontaminate(self): |
| 55 | return True |
| 56 | |
| 57 | def doc_to_decontamination_query(self, doc): |
| 58 | return doc["context"] |
| 59 | |
| 60 | def doc_to_target(self, doc): |
| 61 | return doc["completion"] |
| 62 | |
| 63 | def construct_requests(self, doc, ctx): |
| 64 | ll, is_prediction = rf.loglikelihood(ctx, doc["completion"]) |
| 65 | return is_prediction |
| 66 | |
| 67 | def process_results(self, doc, results): |
| 68 | (is_prediction,) = results |
| 69 | return {"acc": is_prediction} |
| 70 | |
| 71 | def aggregation(self): |
| 72 | return { |
| 73 | "acc": mean, |
| 74 | } |
| 75 | |
| 76 | def higher_is_better(self): |
| 77 | return {"acc": True} |
| 78 | |
| 79 | |
| 80 | class Arithmetic2DPlus(Arithmetic): |
nothing calls this directly
no outgoing calls
no test coverage detected