Converts text/images inputs into tensors that can be used in the forward method for the a model
| 274 | |
| 275 | @dataclasses.dataclass |
| 276 | class MultiModalPreprocessor: |
| 277 | """ |
| 278 | Converts text/images inputs into tensors that can be used in the forward method |
| 279 | for the a model |
| 280 | """ |
| 281 | tokenizer: Any |
| 282 | loss_token_weighting: Optional[str] = None |
| 283 | |
| 284 | # How to crops/resize images |
| 285 | normalize: str = "openai" |
| 286 | crop_mode: str = "resize" |
| 287 | max_crops: int = 6 |
| 288 | overlap_margins: Tuple[int, int] = (4, 4) |
| 289 | resize: str = "default" |
| 290 | use_col_tokens: bool = True |
| 291 | |
| 292 | # Data about the ViT and connector we need when deciding the crops |
| 293 | base_image_input_size: Tuple[int, int] = (336, 336) |
| 294 | image_pooling_w: int = 2 |
| 295 | image_pooling_h: int = 2 |
| 296 | image_token_length_w: int = 12 |
| 297 | image_token_length_h: int = 12 |
| 298 | image_patch_size: int = 14 |
| 299 | image_padding_mask: Union[bool, int] = False |
| 300 | pad_value: float = 0 |
| 301 | |
| 302 | image_patch_token_id: int = dataclasses.field(init=False) |
| 303 | image_col_token_id: int = dataclasses.field(init=False) |
| 304 | image_start_token_id: int = dataclasses.field(init=False) |
| 305 | image_end_token_id: int = dataclasses.field(init=False) |
| 306 | |
| 307 | def __post_init__(self): |
| 308 | special_tokens = get_special_token_ids(self.tokenizer) |
| 309 | self.image_end_token_id = special_tokens[tokenizer.DEFAULT_IM_END_TOKEN] |
| 310 | self.image_start_token_id = special_tokens[tokenizer.DEFAULT_IM_START_TOKEN] |
| 311 | self.image_col_token_id = special_tokens[tokenizer.DEFAULT_IM_COL_TOKEN] |
| 312 | self.image_patch_token_id = special_tokens[tokenizer.DEFAULT_IMAGE_PATCH_TOKEN] |
| 313 | self.image_prompt_token_id = special_tokens[tokenizer.IMAGE_PROMPT] |
| 314 | |
| 315 | def _normalize(self, image): |
| 316 | if self.normalize == "openai": |
| 317 | image -= np.array(OPENAI_CLIP_MEAN, dtype=np.float32)[None, None, :] |
| 318 | image /= np.array(OPENAI_CLIP_STD, dtype=np.float32)[None, None, :] |
| 319 | elif self.normalize == "siglip": |
| 320 | image = np.asarray(-1.0, dtype=np.float32) + image * np.asarray(2.0, dtype=np.float32) |
| 321 | elif self.normalize == "dino": |
| 322 | image -= np.array([0.485, 0.456, 0.406], dtype=np.float32)[None, None, :] |
| 323 | image /= np.array([0.229, 0.224, 0.225], dtype=np.float32)[None, None, :] |
| 324 | else: |
| 325 | raise NotImplementedError(self.normalize) |
| 326 | return image |
| 327 | |
| 328 | def resize_image(self, image, output_size, is_training, rng): |
| 329 | if self.resize == "siglip": |
| 330 | return siglip_resize_and_pad(image, output_size) |
| 331 | elif self.resize == "dino": |
| 332 | return dino_resize_and_pad(image, output_size) |
| 333 | elif self.resize == "metaclip": |
no outgoing calls