(self, x: torch.Tensor, *args, **kwargs)
| 154 | return self._patch_size |
| 155 | |
| 156 | def forward(self, x: torch.Tensor, *args, **kwargs): |
| 157 | out_h = x.shape[-2] // self._patch_size |
| 158 | out_w = x.shape[-1] // self._patch_size |
| 159 | |
| 160 | extra = dict() |
| 161 | |
| 162 | if self._is_dynamic: |
| 163 | pixel_values = rearrange(x, 'b c (h p1) (w p2) -> b (h w) (p1 p2 c)', |
| 164 | p1=self._patch_size, p2=self._patch_size, |
| 165 | h=out_h, w=out_w) |
| 166 | mask = self.mask.expand(*pixel_values.shape[:2]) |
| 167 | shapes = torch.tensor([(out_h, out_w)] * pixel_values.shape[0], dtype=torch.int64, device=x.device) |
| 168 | |
| 169 | extra = dict(attention_mask=mask, spatial_shapes=shapes) |
| 170 | else: |
| 171 | pixel_values = x |
| 172 | |
| 173 | output = self.inner.vision_model(pixel_values=pixel_values, return_dict=True, **extra) |
| 174 | |
| 175 | summary = output.pooler_output |
| 176 | features = output.last_hidden_state |
| 177 | |
| 178 | if kwargs.get('feature_fmt', None) == 'NCHW': |
| 179 | features = rearrange(features, 'b (h w) c -> b c h w', h=out_h, w=out_w) |
| 180 | |
| 181 | return self._wrap_output(summary, features) |
| 182 | |
| 183 | def encode_text(self, text, normalize: bool = False): |
| 184 | output = self.inner.text_model(**text, return_dict=True) |
nothing calls this directly
no test coverage detected