| 1754 | return [] |
| 1755 | |
| 1756 | def _encode(self, example: Dict[str, Any], **kwargs) -> Tuple[Dict[str, Any], Dict[str, Any]]: |
| 1757 | inputs, _ = super()._encode(example) |
| 1758 | if len(inputs) == 0: |
| 1759 | return inputs, {} |
| 1760 | image = example.get('images') or [] |
| 1761 | inputs.pop('loss_scale', None) |
| 1762 | inputs2 = self.tokenizer.build_conversation_input_ids( |
| 1763 | self.tokenizer, query=example['query'], history=example.get('history'), images=image) |
| 1764 | image_token_len = inputs2['token_type_ids'].sum().item() |
| 1765 | input_ids = inputs['input_ids'] |
| 1766 | labels = inputs['labels'] |
| 1767 | inputs['token_type_ids'] = [0] + [1] * image_token_len + [0] * len(input_ids[1:]) |
| 1768 | inputs['input_ids'] = input_ids[:1] + [self.tokenizer.pad_token_id] * image_token_len + input_ids[1:] |
| 1769 | if labels is not None: |
| 1770 | inputs['labels'] = labels[:1] + [-100] * image_token_len + labels[1:] |
| 1771 | if len(image) > 0: |
| 1772 | inputs['images'] = [[img.to(dtype=kwargs['dtype'])] for img in inputs2['images']] |
| 1773 | if 'cross_images' in inputs2: |
| 1774 | # is cogagent |
| 1775 | inputs['cross_images'] = [[cross_img.to(dtype=kwargs['dtype'])] for cross_img in inputs2['cross_images']] |
| 1776 | return inputs, {} |
| 1777 | |
| 1778 | def data_collator(self, batch: List[Dict[str, Any]], padding_to: Optional[int] = None) -> Dict[str, Any]: |
| 1779 | res = super().data_collator(batch, padding_to) |