Preprocess the data by tokenizing.
(
sources: Sequence[str],
targets: Sequence[str],
tokenizer: transformers.PreTrainedTokenizer,
)
| 158 | |
| 159 | |
| 160 | def preprocess( |
| 161 | sources: Sequence[str], |
| 162 | targets: Sequence[str], |
| 163 | tokenizer: transformers.PreTrainedTokenizer, |
| 164 | ) -> Dict: |
| 165 | """Preprocess the data by tokenizing.""" |
| 166 | examples = [s + t for s, t in zip(sources, targets)] |
| 167 | examples_tokenized, sources_tokenized = [_tokenize_fn(strings, tokenizer) for strings in (examples, sources)] |
| 168 | input_ids = examples_tokenized["input_ids"] |
| 169 | labels = copy.deepcopy(input_ids) |
| 170 | # for label, source_len in zip(labels, sources_tokenized["input_ids_lens"]): |
| 171 | # label[:source_len] = IGNORE_INDEX |
| 172 | return dict(input_ids=input_ids, labels=labels) |
| 173 | |
| 174 | |
| 175 | class SupervisedDataset(Dataset): |
no test coverage detected