Only used by LegacyDataset
(self, tokenizer, line, max_length, pad_to_max_length=True, return_tensors="pt")
| 222 | } |
| 223 | |
| 224 | def encode_line(self, tokenizer, line, max_length, pad_to_max_length=True, return_tensors="pt"): |
| 225 | """Only used by LegacyDataset""" |
| 226 | return tokenizer( |
| 227 | [line], |
| 228 | max_length=max_length, |
| 229 | padding="max_length" if pad_to_max_length else None, |
| 230 | truncation=True, |
| 231 | return_tensors=return_tensors, |
| 232 | **self.dataset_kwargs, |
| 233 | ) |
| 234 | |
| 235 | def collate_fn(self, batch) -> Dict[str, torch.Tensor]: |
| 236 | input_ids = torch.stack([x["input_ids"] for x in batch]) |