(examples)
| 25 | ): |
| 26 | |
| 27 | def tokenization(examples): |
| 28 | sources = [] |
| 29 | targets = [] |
| 30 | prompt = PROMPT_TEMPLATE |
| 31 | for instruction, input, output in zip(examples['instruction'],examples['input'],examples['output']): |
| 32 | if input is not None and input !="": |
| 33 | instruction = instruction+'\n'+input |
| 34 | source = prompt.format_map({'instruction':instruction}) |
| 35 | source = instruction |
| 36 | target = f"{output}{tokenizer.eos_token}" |
| 37 | |
| 38 | sources.append(source) |
| 39 | targets.append(target) |
| 40 | |
| 41 | tokenized_sources = tokenizer(sources,return_attention_mask=False) |
| 42 | tokenized_targets = tokenizer(targets,return_attention_mask=False,add_special_tokens=False) |
| 43 | |
| 44 | all_input_ids = [] |
| 45 | all_labels = [] |
| 46 | for s,t in zip(tokenized_sources['input_ids'],tokenized_targets['input_ids']): |
| 47 | input_ids = torch.LongTensor(s + t)[:max_seq_length] |
| 48 | labels = torch.LongTensor([IGNORE_INDEX] * len(s) + t)[:max_seq_length] |
| 49 | assert len(input_ids) == len(labels) |
| 50 | all_input_ids.append(input_ids) |
| 51 | all_labels.append(labels) |
| 52 | |
| 53 | results = {'input_ids':all_input_ids, 'labels': all_labels} |
| 54 | return results |
| 55 | |
| 56 | |
| 57 | logging.warning("building dataset...") |
nothing calls this directly
no outgoing calls
no test coverage detected