Preprocess the data by tokenizing.
(self, sources, targets, tokenizer)
| 85 | return self.gpt_convert_example_to_feature(raw_sample) |
| 86 | |
| 87 | def preprocess(self, sources, targets, tokenizer): |
| 88 | """Preprocess the data by tokenizing.""" |
| 89 | examples = [s + t for s, t in zip(sources, targets)] |
| 90 | examples_tokenized, sources_tokenized = [ |
| 91 | self.tokenize(strings, tokenizer) |
| 92 | for strings in (examples, sources) |
| 93 | ] |
| 94 | input_ids = examples_tokenized['input_ids'] |
| 95 | labels = copy.deepcopy(input_ids) |
| 96 | for label, source_len in zip(labels, |
| 97 | sources_tokenized['input_ids_lens']): |
| 98 | label[:source_len] = self.IGNORE_INDEX |
| 99 | return dict(input_ids=input_ids, labels=labels) |
| 100 | |
| 101 | def tokenize(self, strings, tokenizer): |
| 102 | """ |