(examples, cutoff_len, tokenizer)
| 175 | return model_inputs |
| 176 | |
| 177 | def pad_sequence(examples, cutoff_len, tokenizer): |
| 178 | max_length = cutoff_len |
| 179 | input_pad_token_id = tokenizer.pad_token_id |
| 180 | label_pad_token_id = IGNORE_INDEX |
| 181 | |
| 182 | for k, v in examples.items(): |
| 183 | if k.endswith("input_ids"): |
| 184 | pad_token_id = input_pad_token_id |
| 185 | elif k.endswith("labels"): |
| 186 | pad_token_id = label_pad_token_id |
| 187 | # shift labels here |
| 188 | for i in range(len(v)): |
| 189 | v[i] = v[i][1:] |
| 190 | elif k.endswith("attention_mask"): |
| 191 | pad_token_id = 0 |
| 192 | elif k.endswith("position_ids"): |
| 193 | pad_token_id = max_length - 1 # pad the max position id |
| 194 | elif k == "images" or k == "videos": |
| 195 | pad_token_id = -1 |
| 196 | continue # TODO: haven't tested multi-modal yet |
| 197 | else: |
| 198 | continue |
| 199 | for i in range(len(v)): |
| 200 | v[i].extend([pad_token_id] * (max_length - len(v[i]))) |
| 201 | examples[k] = v |
| 202 | |
| 203 | return examples |
| 204 | |
| 205 | def preprocess_sp_dataset(seq_ids, world_size, sequence_parallel_mode): |
| 206 | if sequence_parallel_mode == "zigzag-ring": |
nothing calls this directly
no outgoing calls
no test coverage detected