| 813 | |
| 814 | @dataclasses.dataclass |
| 815 | class Preprocessor: |
| 816 | formater: DataFormatter |
| 817 | mm_preprocessor: MultiModalPreprocessor |
| 818 | for_inference: bool = False |
| 819 | is_training: bool = False |
| 820 | shuffle_messages: bool = False |
| 821 | include_image: bool = False |
| 822 | require_image_features: bool = False |
| 823 | |
| 824 | def __call__(self, example, rng=np.random): |
| 825 | example = dict(example) |
| 826 | if "image" in example: |
| 827 | try: |
| 828 | image = load_image(example["image"]) |
| 829 | except Exception as e: |
| 830 | raise ValueError(f"Could not load image: {example['image']}") |
| 831 | else: |
| 832 | example["image"] = image |
| 833 | else: |
| 834 | image = None |
| 835 | |
| 836 | messages, formatter_metadata = self.formater(example, self.is_training, self.for_inference, rng) |
| 837 | if self.shuffle_messages and isinstance(messages[0], list): |
| 838 | # If there are multiple conversations for this example, shuffle their order |
| 839 | # This might matter if we truncate the tokens to a max sequence length |
| 840 | rng.shuffle(messages) |
| 841 | batch = self.mm_preprocessor( |
| 842 | image, |
| 843 | messages, |
| 844 | weight=example.get("weight"), |
| 845 | rng=rng, |
| 846 | is_training=self.is_training, |
| 847 | require_image_features=self.require_image_features |
| 848 | ) |
| 849 | if formatter_metadata is None: |
| 850 | formatter_metadata = {} |
| 851 | if self.include_image and image is not None: |
| 852 | formatter_metadata["image"] = image |
| 853 | if image is not None: |
| 854 | h, w = image.shape[:2] |
| 855 | formatter_metadata["image_size"] = (w, h) |
| 856 | if "metadata" in example or formatter_metadata: |
| 857 | metadata = example.get("metadata", {}) |
| 858 | if formatter_metadata: |
| 859 | metadata.update(formatter_metadata) |
| 860 | batch["metadata"] = metadata |
| 861 | return batch |
| 862 | |
| 863 | @property |
| 864 | def tokenizer(self): |
| 865 | return self.mm_preprocessor.tokenizer |
no outgoing calls
no test coverage detected