(self, inputs)
| 160 | |
| 161 | |
| 162 | def vace(self, inputs): |
| 163 | self.pipe.load_models_to_device(["vae"]) |
| 164 | vace_video = self.pipe.preprocess_video(inputs["vace_video"]) |
| 165 | vace_video_mask = self.pipe.preprocess_video(inputs["vace_video_mask"], min_value=0, max_value=1) |
| 166 | |
| 167 | inactive = vace_video * (1 - vace_video_mask) + 0 * vace_video_mask |
| 168 | reactive = vace_video * vace_video_mask + 0 * (1 - vace_video_mask) |
| 169 | inactive = self.pipe.vae.encode(inactive, device=self.pipe.device, tiled=inputs["tiled"], tile_size=None, tile_stride=None).to(dtype=self.pipe.torch_dtype, device=self.pipe.device) |
| 170 | reactive = self.pipe.vae.encode(reactive, device=self.pipe.device, tiled=inputs["tiled"], tile_size=None, tile_stride=None).to(dtype=self.pipe.torch_dtype, device=self.pipe.device) |
| 171 | vace_video_latents = torch.concat((inactive, reactive), dim=1) |
| 172 | vace_mask_latents = rearrange(vace_video_mask[0,0], "T (H P) (W Q) -> 1 (P Q) T H W", P=8, Q=8) |
| 173 | vace_mask_latents = torch.nn.functional.interpolate(vace_mask_latents, size=((vace_mask_latents.shape[2] + 3) // 4, vace_mask_latents.shape[3], vace_mask_latents.shape[4]), mode='nearest-exact') |
| 174 | |
| 175 | vace_reference_image = self.pipe.preprocess_video([inputs["vace_reference_image"]]) |
| 176 | bs, c, f, h, w = vace_reference_image.shape |
| 177 | new_vace_ref_images = [] |
| 178 | for j in range(f): |
| 179 | new_vace_ref_images.append(vace_reference_image[0, :, j:j+1]) |
| 180 | vace_reference_image = new_vace_ref_images |
| 181 | vace_reference_latents = self.pipe.vae.encode(vace_reference_image, device=self.pipe.device, tiled=inputs["tiled"], tile_size=None, tile_stride=None).to(dtype=self.pipe.torch_dtype, device=self.pipe.device) |
| 182 | vace_reference_latents = torch.concat((vace_reference_latents, torch.zeros_like(vace_reference_latents)), dim=1) |
| 183 | vace_reference_latents = [u.unsqueeze(0) for u in vace_reference_latents] |
| 184 | |
| 185 | vace_video_latents = torch.concat((*vace_reference_latents, vace_video_latents), dim=2) |
| 186 | vace_mask_latents = torch.concat((torch.zeros_like(vace_mask_latents[:, :, :f]), vace_mask_latents), dim=2) |
| 187 | vace_context = torch.concat((vace_video_latents, vace_mask_latents), dim=1) |
| 188 | |
| 189 | inputs["vace_context"] = vace_context |
| 190 | return inputs |
| 191 | |
| 192 | |
| 193 | def trainable_modules(self): |
no test coverage detected