| 118 | |
| 119 | @torch.no_grad() |
| 120 | def forward_vision_encoder(self, images): |
| 121 | dtype = next(self.pipeline.vision_encoder.parameters()).dtype |
| 122 | image_pil = [v2.functional.to_pil_image(images[i]) for i in range(images.shape[0])] |
| 123 | image_pt = self.pipeline.feature_extractor_clip(images=image_pil, return_tensors="pt").pixel_values |
| 124 | image_pt = image_pt.to(device=self.device, dtype=dtype) |
| 125 | global_embeds = self.pipeline.vision_encoder(image_pt, output_hidden_states=False).image_embeds |
| 126 | global_embeds = global_embeds.unsqueeze(-2) |
| 127 | |
| 128 | encoder_hidden_states = self.pipeline._encode_prompt("", self.device, 1, False)[0] |
| 129 | ramp = global_embeds.new_tensor(self.pipeline.config.ramping_coefficients).unsqueeze(-1) |
| 130 | encoder_hidden_states = encoder_hidden_states + global_embeds * ramp |
| 131 | |
| 132 | return encoder_hidden_states |
| 133 | |
| 134 | @torch.no_grad() |
| 135 | def encode_condition_image(self, images): |