(self, instances: Sequence[Dict])
| 93 | tokenizer: transformers.PreTrainedTokenizer |
| 94 | |
| 95 | def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]: |
| 96 | input_ids, labels = tuple([instance[key] for instance in instances] for key in ("input_ids", "labels")) |
| 97 | |
| 98 | input_ids = torch.nn.utils.rnn.pad_sequence( |
| 99 | input_ids, batch_first=True, padding_value=self.tokenizer.pad_token_id |
| 100 | ) |
| 101 | labels = torch.nn.utils.rnn.pad_sequence(labels, batch_first=True, padding_value=-100) |
| 102 | return dict( |
| 103 | input_ids=input_ids, |
| 104 | labels=labels, |
| 105 | attention_mask=input_ids.ne(self.tokenizer.pad_token_id), |
| 106 | ) |
| 107 | |
| 108 | def fault_tolerance_data_collator(features: List) -> Dict[str, Any]: |
| 109 | if not isinstance(features[0], Mapping): |
nothing calls this directly
no outgoing calls
no test coverage detected