Call tokenizer on src and tgt_lines
(self, index)
| 203 | |
| 204 | class LegacySeq2SeqDataset(AbstractSeq2SeqDataset): |
| 205 | def __getitem__(self, index) -> Dict[str, torch.Tensor]: |
| 206 | """Call tokenizer on src and tgt_lines""" |
| 207 | index = index + 1 # linecache starts at 1 |
| 208 | source_line = self.prefix + linecache.getline(str(self.src_file), index).rstrip("\n") |
| 209 | tgt_line = linecache.getline(str(self.tgt_file), index).rstrip("\n") |
| 210 | assert source_line, f"empty source line for index {index}" |
| 211 | assert tgt_line, f"empty tgt line for index {index}" |
| 212 | source_inputs = self.encode_line(self.tokenizer, source_line, self.max_source_length) |
| 213 | target_inputs = self.encode_line(self.tokenizer, tgt_line, self.max_target_length) |
| 214 | |
| 215 | source_ids = source_inputs["input_ids"].squeeze() |
| 216 | target_ids = target_inputs["input_ids"].squeeze() |
| 217 | src_mask = source_inputs["attention_mask"].squeeze() |
| 218 | return { |
| 219 | "input_ids": source_ids, |
| 220 | "attention_mask": src_mask, |
| 221 | "labels": target_ids, |
| 222 | } |
| 223 | |
| 224 | def encode_line(self, tokenizer, line, max_length, pad_to_max_length=True, return_tensors="pt"): |
| 225 | """Only used by LegacyDataset""" |
nothing calls this directly
no test coverage detected