| 31 | |
| 32 | |
| 33 | class ANLIBase(Task): |
| 34 | VERSION = 0 |
| 35 | DATASET_PATH = "anli" |
| 36 | DATASET_NAME = None |
| 37 | SPLIT = None |
| 38 | |
| 39 | def has_training_docs(self): |
| 40 | return True |
| 41 | |
| 42 | def has_validation_docs(self): |
| 43 | return True |
| 44 | |
| 45 | def has_test_docs(self): |
| 46 | return True |
| 47 | |
| 48 | def training_docs(self): |
| 49 | if self.has_training_docs(): |
| 50 | if self._training_docs is None: |
| 51 | self._training_docs = list(self.dataset["train_r" + str(self.SPLIT)]) |
| 52 | return self._training_docs |
| 53 | |
| 54 | def validation_docs(self): |
| 55 | if self.has_validation_docs(): |
| 56 | return self.dataset["dev_r" + str(self.SPLIT)] |
| 57 | |
| 58 | def test_docs(self): |
| 59 | if self.has_test_docs(): |
| 60 | return self.dataset["test_r" + str(self.SPLIT)] |
| 61 | |
| 62 | def doc_to_text(self, doc): |
| 63 | # OA does this a bit weirdly: they prepend "anli 1: anli 1: " to the beginning |
| 64 | # of the prompt (yes, repeating it!). also, " True, False, or Neither?" is directly |
| 65 | # appended onto the question, with no "Answer:" or even a newline. Do we *really* |
| 66 | # want to do it exactly as OA did? |
| 67 | return ( |
| 68 | doc["premise"] |
| 69 | + "\nQuestion: " |
| 70 | + doc["hypothesis"] |
| 71 | + " True, False, or Neither?\nAnswer:" |
| 72 | ) |
| 73 | |
| 74 | def should_decontaminate(self): |
| 75 | return True |
| 76 | |
| 77 | def doc_to_decontamination_query(self, doc): |
| 78 | return doc["premise"] |
| 79 | |
| 80 | def doc_to_target(self, doc): |
| 81 | # True = entailment |
| 82 | # False = contradiction |
| 83 | # Neither = neutral |
| 84 | return " " + ["True", "Neither", "False"][doc["label"]] |
| 85 | |
| 86 | def construct_requests(self, doc, ctx): |
| 87 | """Uses RequestFactory to construct Requests and returns an iterable of |
| 88 | Requests which will be sent to the LM. |
| 89 | |
| 90 | :param doc: |
nothing calls this directly
no outgoing calls
no test coverage detected