(
self, data: Union[Image.Image, tuple,
Dict[str, Any]])
| 393 | return self._image_map[path] |
| 394 | |
| 395 | def __call__( |
| 396 | self, data: Union[Image.Image, tuple, |
| 397 | Dict[str, Any]]) -> Dict[str, Any]: |
| 398 | self.cfg = Config.from_file( |
| 399 | osp.join(self.model_dir, ModelFile.CONFIGURATION)) |
| 400 | |
| 401 | if isinstance(data, (Image.Image, str)): |
| 402 | image = data |
| 403 | elif isinstance(data, tuple): |
| 404 | image = data[0] |
| 405 | else: |
| 406 | image = data['image'] |
| 407 | index = 0 |
| 408 | if isinstance(image, str): |
| 409 | image, index = self.image_open(image) |
| 410 | image = image.convert('RGB') |
| 411 | image = self.patch_resize_transform(image) |
| 412 | question = '' if self.cfg.task == Tasks.image_captioning \ |
| 413 | else data[1 if isinstance(data, tuple) |
| 414 | else ('text' if 'text' in data else 'question')] |
| 415 | question = self.tokenizer( |
| 416 | question.lower(), |
| 417 | padding='max_length', |
| 418 | truncation=True, |
| 419 | max_length=self.tokenizer_max_length, |
| 420 | return_tensors='pt') |
| 421 | |
| 422 | if self.mode == ModeKeys.INFERENCE: |
| 423 | image = torch.stack([image], dim=0) |
| 424 | return {'image': image, 'question': question} |
| 425 | else: |
| 426 | answer = data['answer'] |
| 427 | answer = self.tokenizer( |
| 428 | answer, |
| 429 | padding='max_length', |
| 430 | truncation=True, |
| 431 | max_length=self.tokenizer_max_length, |
| 432 | return_tensors='pt') |
| 433 | output = { |
| 434 | 'image': image, |
| 435 | 'question_input_ids': question.input_ids.squeeze(), |
| 436 | 'question_attention_mask': question.attention_mask.squeeze(), |
| 437 | 'answer_input_ids': answer.input_ids.squeeze(), |
| 438 | 'answer_attention_mask': answer.attention_mask.squeeze(), |
| 439 | } |
| 440 | if self.cfg.task == Tasks.image_text_retrieval: |
| 441 | output['index'] = index |
| 442 | return output |
| 443 | |
| 444 | |
| 445 | @PREPROCESSORS.register_module( |
nothing calls this directly
no test coverage detected