| 52 | return True |
| 53 | |
| 54 | def _collate_data(self, set): |
| 55 | if set in self.cache: |
| 56 | return self.cache[set] |
| 57 | # One big issue with HF's implementation of this dataset: it makes a |
| 58 | # separate document for each question; meanwhile, in the GPT3 paper it |
| 59 | # is shown that one document is made per passage. |
| 60 | |
| 61 | r = collections.defaultdict(list) |
| 62 | for item in datasets.load_dataset( |
| 63 | path=self.DATASET_PATH, name=self.DATASET_NAME |
| 64 | )[set]: |
| 65 | r[item["article"]].append(item) |
| 66 | |
| 67 | res = list( |
| 68 | r.values() |
| 69 | >> each( |
| 70 | lambda x: { |
| 71 | "article": x[0]["article"], |
| 72 | "problems": x |
| 73 | >> each( |
| 74 | lambda y: { |
| 75 | "question": y["question"], |
| 76 | "answer": y["answer"], |
| 77 | "options": y["options"], |
| 78 | } |
| 79 | ), |
| 80 | } |
| 81 | ) |
| 82 | ) |
| 83 | |
| 84 | self.cache[set] = res |
| 85 | return res |
| 86 | |
| 87 | def training_docs(self): |
| 88 | return self._collate_data("train") |