(self, stage)
| 131 | self.dataset_reader = dataset_reader |
| 132 | |
| 133 | def setup(self, stage): |
| 134 | self.train_datasets = self.dataset_reader.read_orig_dataset("train") |
| 135 | self.base_templates = self.dataset_reader.get_template() |
| 136 | self.train_datasets_withtemplate = [] |
| 137 | for index, train_dataset in enumerate(self.train_datasets): |
| 138 | self.train_datasets_withtemplate.append( |
| 139 | PretrainDatasetWithTemplate(train_dataset, self.base_templates[index], self.tokenizer) |
| 140 | ) |
| 141 | |
| 142 | self.train_dataset = torch.utils.data.ConcatDataset(self.train_datasets_withtemplate) |
| 143 | |
| 144 | def train_dataloader(self): |
| 145 | return torch.utils.data.DataLoader( |
nothing calls this directly
no test coverage detected