(self, text)
| 337 | return self.visual(image.type(self.dtype)) |
| 338 | |
| 339 | def encode_text(self, text): |
| 340 | x = self.token_embedding(text).type(self.dtype) # [batch_size, n_ctx, d_model] |
| 341 | |
| 342 | x = x + self.positional_embedding.type(self.dtype) |
| 343 | x = x.permute(1, 0, 2) # NLD -> LND |
| 344 | x = self.transformer(x) |
| 345 | x = x.permute(1, 0, 2) # LND -> NLD |
| 346 | x = self.ln_final(x).type(self.dtype) |
| 347 | |
| 348 | # x.shape = [batch_size, n_ctx, transformer.width] |
| 349 | # take features from the eot embedding (eot_token is the highest number in each sequence) |
| 350 | x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.text_projection |
| 351 | |
| 352 | return x |
| 353 | |
| 354 | def forward(self, image, text): |
| 355 | image_features = self.encode_image(image) |
no outgoing calls
no test coverage detected