(examples, tokenizer, max_seq_length)
| 83 | return conversations, conv |
| 84 | |
| 85 | def process_func(examples, tokenizer, max_seq_length): |
| 86 | conversations, conv = format_inputs(examples['conversations']) |
| 87 | model_inputs = tokenizer( |
| 88 | conversations, |
| 89 | max_length=max_seq_length, |
| 90 | padding="max_length", |
| 91 | truncation=True, |
| 92 | return_tensors="pt", |
| 93 | ) |
| 94 | |
| 95 | model_inputs.pop("token_type_ids", None) |
| 96 | # If we are padding here, replace all tokenizer.pad_token_id in the labels by -100 when we want to ignore |
| 97 | # padding in the loss. |
| 98 | targets = model_inputs["input_ids"].clone() |
| 99 | |
| 100 | # Mask targets |
| 101 | sep = conv.sep + conv.roles[1] + ": " |
| 102 | for conversation, target in zip(conversations, targets): |
| 103 | total_len = int(target.ne(tokenizer.pad_token_id).sum()) |
| 104 | |
| 105 | turns = conversation.split(conv.sep2) |
| 106 | cur_len = 1 |
| 107 | target[:cur_len] = IGNORE_INDEX |
| 108 | for i, turn in enumerate(turns): |
| 109 | if turn == "": |
| 110 | break |
| 111 | turn_len = len(tokenizer(turn).input_ids) |
| 112 | |
| 113 | parts = turn.split(sep) |
| 114 | if len(parts) != 2: |
| 115 | break |
| 116 | parts[0] += sep |
| 117 | |
| 118 | # "-2" is hardcoded for the Llama tokenizer to make the offset correct. |
| 119 | instruction_len = len(tokenizer(parts[0]).input_ids) - 2 |
| 120 | |
| 121 | if i != 0 and not tokenizer.legacy: |
| 122 | # The legacy and non-legacy modes handle special tokens differently |
| 123 | instruction_len -= 1 |
| 124 | |
| 125 | # Ignore the user instructions |
| 126 | target[cur_len: cur_len + instruction_len] = IGNORE_INDEX |
| 127 | cur_len += turn_len |
| 128 | |
| 129 | if i != 0 and not tokenizer.legacy: |
| 130 | # The legacy and non-legacy modes handle special tokens differently |
| 131 | cur_len -= 1 |
| 132 | |
| 133 | target[cur_len:] = IGNORE_INDEX |
| 134 | |
| 135 | if cur_len < tokenizer.model_max_length: |
| 136 | if cur_len != total_len: |
| 137 | target[:] = IGNORE_INDEX |
| 138 | |
| 139 | model_inputs["labels"] = targets |
| 140 | return model_inputs |
| 141 | |
| 142 | class BaseDataset(Dataset): |
nothing calls this directly
no test coverage detected