| 358 | tokenizer: transformers.PreTrainedTokenizer |
| 359 | |
| 360 | def pad_sequence(self, input_ids, batch_first, padding_value): |
| 361 | if self.tokenizer.padding_side == "left": |
| 362 | input_ids = [torch.flip(_input_ids, [0]) for _input_ids in input_ids] |
| 363 | input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=batch_first, padding_value=padding_value) |
| 364 | if self.tokenizer.padding_side == "left": |
| 365 | input_ids = torch.flip(input_ids, [1]) |
| 366 | return input_ids |
| 367 | |
| 368 | def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]: |
| 369 | input_ids, labels = tuple([instance[key] for instance in instances] for key in ("input_ids", "labels")) |