| 542 | return [[-100]] |
| 543 | |
| 544 | def _encode(self, example: Dict[str, Any], **kwargs) -> Tuple[Dict[str, Any], Dict[str, Any]]: |
| 545 | inputs, _ = super()._encode(example) |
| 546 | if len(inputs) == 0: |
| 547 | return inputs, {} |
| 548 | input_ids = inputs['input_ids'] |
| 549 | labels = inputs['labels'] |
| 550 | idx_list = _findall(input_ids, -100) |
| 551 | if idx_list: |
| 552 | idx = idx_list[0] |
| 553 | image = example.get('images')[0] |
| 554 | placeholder = '<|begin_of_image|><|endoftext|><|end_of_image|>' |
| 555 | placeholder_id = self.tokenizer.encode(placeholder, add_special_tokens=False) |
| 556 | input_ids = (input_ids[:idx] + placeholder_id + input_ids[idx + 1:]) |
| 557 | if labels is not None: |
| 558 | labels = (labels[:idx] + [-100] * len(placeholder_id) + labels[idx + 1:]) |
| 559 | messages = example['messages'] |
| 560 | messages[0]['image'] = image |
| 561 | inputs2: Dict[str, Any] = self.tokenizer.apply_chat_template(messages, return_dict=True) |
| 562 | inputs['images'] = inputs2['images'] |
| 563 | inputs['input_ids'] = input_ids |
| 564 | inputs['labels'] = labels |
| 565 | return inputs, {} |
| 566 | |
| 567 | def data_collator(self, batch: List[Dict[str, Any]], padding_to: Optional[int] = None) -> Dict[str, Any]: |
| 568 | res = super().data_collator(batch, padding_to) |