(examples)
| 508 | padding = "max_length" if data_args.pad_to_max_length else False |
| 509 | |
| 510 | def husky_processor(examples): |
| 511 | processor = partial( |
| 512 | process_func, |
| 513 | tokenizer=tokenizer, |
| 514 | max_seq_length=data_args.max_seq_length, |
| 515 | ) |
| 516 | model_inputs = processor(examples) |
| 517 | return model_inputs |
| 518 | |
| 519 | # Data collator |
| 520 | label_pad_token_id = IGNORE_INDEX if data_args.ignore_pad_token_for_loss else tokenizer.pad_token_id |
nothing calls this directly
no outgoing calls
no test coverage detected