| 196 | return utils.create_video(result, fps, path=save_path, watermark=gradio_utils.logo_name_to_path(watermark)) |
| 197 | |
| 198 | def process_controlnet_depth(self, |
| 199 | video_path, |
| 200 | prompt, |
| 201 | chunk_size=8, |
| 202 | watermark='Picsart AI Research', |
| 203 | merging_ratio=0.0, |
| 204 | num_inference_steps=20, |
| 205 | controlnet_conditioning_scale=1.0, |
| 206 | guidance_scale=9.0, |
| 207 | seed=42, |
| 208 | eta=0.0, |
| 209 | resolution=512, |
| 210 | use_cf_attn=True, |
| 211 | save_path=None): |
| 212 | print("Module Depth") |
| 213 | video_path = gradio_utils.edge_path_to_video_path(video_path) |
| 214 | if self.model_type != ModelType.ControlNetDepth: |
| 215 | controlnet = ControlNetModel.from_pretrained( |
| 216 | "lllyasviel/sd-controlnet-depth") |
| 217 | self.set_model(ModelType.ControlNetDepth, |
| 218 | model_id="runwayml/stable-diffusion-v1-5", controlnet=controlnet) |
| 219 | self.pipe.scheduler = DDIMScheduler.from_config( |
| 220 | self.pipe.scheduler.config) |
| 221 | if use_cf_attn: |
| 222 | self.pipe.unet.set_attn_processor( |
| 223 | processor=self.controlnet_attn_proc) |
| 224 | self.pipe.controlnet.set_attn_processor( |
| 225 | processor=self.controlnet_attn_proc) |
| 226 | |
| 227 | added_prompt = 'best quality, extremely detailed' |
| 228 | negative_prompts = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality' |
| 229 | |
| 230 | video, fps = utils.prepare_video( |
| 231 | video_path, resolution, self.device, self.dtype, False) |
| 232 | control = utils.pre_process_depth( |
| 233 | video).to(self.device).to(self.dtype) |
| 234 | |
| 235 | # depth_map_to_save = list(rearrange(control, 'f c w h -> f w h c').cpu().detach().numpy()) |
| 236 | # _ = utils.create_video(depth_map_to_save, 4, path="ddxk.mp4", watermark=None) |
| 237 | |
| 238 | f, _, h, w = video.shape |
| 239 | self.generator.manual_seed(seed) |
| 240 | latents = torch.randn((1, 4, h//8, w//8), dtype=self.dtype, |
| 241 | device=self.device, generator=self.generator) |
| 242 | latents = latents.repeat(f, 1, 1, 1) |
| 243 | result = self.inference(image=control, |
| 244 | prompt=prompt + ', ' + added_prompt, |
| 245 | height=h, |
| 246 | width=w, |
| 247 | negative_prompt=negative_prompts, |
| 248 | num_inference_steps=num_inference_steps, |
| 249 | guidance_scale=guidance_scale, |
| 250 | controlnet_conditioning_scale=controlnet_conditioning_scale, |
| 251 | eta=eta, |
| 252 | latents=latents, |
| 253 | seed=seed, |
| 254 | output_type='numpy', |
| 255 | split_to_chunks=True, |