| 786 | vision_config = nemo_config["mm_cfg"]["vision_encoder"] |
| 787 | |
| 788 | class VisionEncoderWrapper(torch.nn.Module): |
| 789 | |
| 790 | def __init__(self, encoder, connector): |
| 791 | super().__init__() |
| 792 | self.encoder = encoder |
| 793 | self.connector = connector |
| 794 | |
| 795 | def forward(self, images): |
| 796 | vision_x = self.encoder(pixel_values=images, |
| 797 | output_hidden_states=True) |
| 798 | vision_x = vision_x.hidden_states[-2] |
| 799 | vision_x = vision_x[:, 1:] |
| 800 | vision_x = self.connector(vision_x) |
| 801 | return vision_x |
| 802 | |
| 803 | vision_path = vision_config["from_pretrained"] |
| 804 | joined_path = os.path.join(os.path.dirname(args.model_path), |
no outgoing calls
no test coverage detected