(
self, data: Union[decord.VideoReader, tuple,
Dict[str, Any]])
| 599 | return frame_indices |
| 600 | |
| 601 | def __call__( |
| 602 | self, data: Union[decord.VideoReader, tuple, |
| 603 | Dict[str, Any]]) -> Dict[str, Any]: |
| 604 | self.cfg = Config.from_file( |
| 605 | osp.join(self.model_dir, ModelFile.CONFIGURATION)) |
| 606 | |
| 607 | if isinstance(data, (decord.VideoReader, str)): |
| 608 | video = data |
| 609 | elif isinstance(data, tuple): |
| 610 | video = data[0] |
| 611 | else: |
| 612 | video = data['video'] |
| 613 | index = 0 |
| 614 | if isinstance(video, str): |
| 615 | video, index = self.video_open(video) |
| 616 | frame_indices = self.sample_frames(self.num_frames, len(video)) |
| 617 | video.seek(0) |
| 618 | video = torch.from_numpy(video.get_batch(frame_indices).asnumpy()) |
| 619 | video = [ |
| 620 | self.patch_resize_transform(Image.fromarray(f)) |
| 621 | for f in video.numpy() |
| 622 | ] |
| 623 | video = torch.stack(video, dim=0) |
| 624 | question = '' if self.cfg.task == Tasks.video_captioning \ |
| 625 | else data[1 if isinstance(data, tuple) |
| 626 | else ('text' if 'text' in data else 'question')] |
| 627 | question = self.tokenizer( |
| 628 | question.lower(), |
| 629 | padding='max_length', |
| 630 | truncation=True, |
| 631 | max_length=self.tokenizer_max_length, |
| 632 | return_tensors='pt') |
| 633 | |
| 634 | if self.mode == ModeKeys.INFERENCE: |
| 635 | video = torch.stack([video], dim=0) |
| 636 | return {'video': video, 'question': question} |
| 637 | else: |
| 638 | answer = data['answer'] |
| 639 | answer = self.tokenizer( |
| 640 | answer, |
| 641 | padding='max_length', |
| 642 | truncation=True, |
| 643 | max_length=self.tokenizer_max_length, |
| 644 | return_tensors='pt') |
| 645 | output = { |
| 646 | 'video': video, |
| 647 | 'question_input_ids': question.input_ids.squeeze(), |
| 648 | 'question_attention_mask': question.attention_mask.squeeze(), |
| 649 | 'answer_input_ids': answer.input_ids.squeeze(), |
| 650 | 'answer_attention_mask': answer.attention_mask.squeeze(), |
| 651 | } |
| 652 | return output |
| 653 | |
| 654 | |
| 655 | @PREPROCESSORS.register_module( |
nothing calls this directly
no test coverage detected