| 27 | |
| 28 | |
| 29 | class ARCEasy(MultipleChoiceTask): |
| 30 | VERSION = 0 |
| 31 | # DATASET_PATH = "/root/model/datasets/ai2_arc/ARC-Easy/1.0.0" |
| 32 | DATASET_NAME = "ARC-Easy" |
| 33 | |
| 34 | def has_training_docs(self): |
| 35 | return True |
| 36 | |
| 37 | def has_validation_docs(self): |
| 38 | return True |
| 39 | |
| 40 | def has_test_docs(self): |
| 41 | return True |
| 42 | |
| 43 | def training_docs(self): |
| 44 | if self._training_docs is None: |
| 45 | self._training_docs = list(map(self._process_doc, self.dataset["train"])) |
| 46 | return self._training_docs |
| 47 | |
| 48 | def validation_docs(self): |
| 49 | return map(self._process_doc, self.dataset["validation"]) |
| 50 | |
| 51 | def test_docs(self): |
| 52 | return map(self._process_doc, self.dataset["test"]) |
| 53 | |
| 54 | def _process_doc(self, doc): |
| 55 | # NOTE: Some `doc["answerKey"]`s are in numeric string format being one |
| 56 | # of {'1', '2', '3', '4', '5'}. We map them back to letters. |
| 57 | num_to_letter = {"1": "A", "2": "B", "3": "C", "4": "D", "5": "E"} |
| 58 | doc["answerKey"] = num_to_letter.get(doc["answerKey"], doc["answerKey"]) |
| 59 | out_doc = { |
| 60 | "id": doc["id"], |
| 61 | "query": "Question: " + doc["question"] + "\nAnswer:", |
| 62 | "choices": doc["choices"]["text"], |
| 63 | "gold": ["A", "B", "C", "D", "E"].index(doc["answerKey"]), |
| 64 | } |
| 65 | return out_doc |
| 66 | |
| 67 | def doc_to_text(self, doc): |
| 68 | return doc["query"] |
| 69 | |
| 70 | def should_decontaminate(self): |
| 71 | return True |
| 72 | |
| 73 | def doc_to_decontamination_query(self, doc): |
| 74 | return doc["query"] |
| 75 | |
| 76 | |
| 77 | class ARCChallenge(ARCEasy): |
nothing calls this directly
no outgoing calls
no test coverage detected