(self, instances: Sequence[Dict])
| 230 | ) |
| 231 | |
| 232 | def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]: |
| 233 | sources = [] |
| 234 | targets = [] |
| 235 | for instance in instances: |
| 236 | source = instance['input_ids'] |
| 237 | target = instance['labels'] |
| 238 | sources.append(source) |
| 239 | targets.append(target) |
| 240 | |
| 241 | data_dict = preprocess(sources, targets, self.tokenizer) |
| 242 | input_ids, labels = data_dict['input_ids'], data_dict['labels'] |
| 243 | |
| 244 | input_ids = torch.nn.utils.rnn.pad_sequence( |
| 245 | input_ids, batch_first=True, padding_value=self.tokenizer.pad_token_id |
| 246 | ) |
| 247 | labels = torch.nn.utils.rnn.pad_sequence(labels, batch_first=True, padding_value=IGNORE_INDEX) |
| 248 | return dict( |
| 249 | input_ids=input_ids, |
| 250 | labels=labels, |
| 251 | attention_mask=input_ids.ne(self.tokenizer.pad_token_id), |
| 252 | ) |
| 253 | |
| 254 | def make_supervised_data_module(tokenizer: transformers.PreTrainedTokenizer, data_args) -> Dict: |
| 255 | """Make dataset and collator for supervised fine-tuning.""" |
nothing calls this directly
no test coverage detected