(self, *args, **kwargs)
| 161 | return doc["input"] |
| 162 | |
| 163 | def download(self, *args, **kwargs): |
| 164 | super().download(*args, **kwargs) |
| 165 | del self.dataset["test"] |
| 166 | for split in self.dataset: |
| 167 | self.dataset[split] = _drop_duplicates_in_input(self.dataset[split]) |
| 168 | if self.PRUNE_TOKENIZERS is not None and self.PRUNE_TOKENIZERS is not None: |
| 169 | self.prune() |
| 170 | |
| 171 | def _get_prune_text(self, sample): |
| 172 | return self.doc_to_text(self._process_doc(sample)[0]) |
nothing calls this directly
no test coverage detected