MCPcopy Create free account
hub / github.com/Picsart-AI-Research/Text2Video-Zero / process_controlnet_depth

Method process_controlnet_depth

model.py:198–259  ·  view source on GitHub ↗
(self,
                                 video_path,
                                 prompt,
                                 chunk_size=8,
                                 watermark='Picsart AI Research',
                                 merging_ratio=0.0,
                                 num_inference_steps=20,
                                 controlnet_conditioning_scale=1.0,
                                 guidance_scale=9.0,
                                 seed=42,
                                 eta=0.0,
                                 resolution=512,
                                 use_cf_attn=True,
                                 save_path=None)

Source from the content-addressed store, hash-verified

196 return utils.create_video(result, fps, path=save_path, watermark=gradio_utils.logo_name_to_path(watermark))
197
198 def process_controlnet_depth(self,
199 video_path,
200 prompt,
201 chunk_size=8,
202 watermark='Picsart AI Research',
203 merging_ratio=0.0,
204 num_inference_steps=20,
205 controlnet_conditioning_scale=1.0,
206 guidance_scale=9.0,
207 seed=42,
208 eta=0.0,
209 resolution=512,
210 use_cf_attn=True,
211 save_path=None):
212 print("Module Depth")
213 video_path = gradio_utils.edge_path_to_video_path(video_path)
214 if self.model_type != ModelType.ControlNetDepth:
215 controlnet = ControlNetModel.from_pretrained(
216 "lllyasviel/sd-controlnet-depth")
217 self.set_model(ModelType.ControlNetDepth,
218 model_id="runwayml/stable-diffusion-v1-5", controlnet=controlnet)
219 self.pipe.scheduler = DDIMScheduler.from_config(
220 self.pipe.scheduler.config)
221 if use_cf_attn:
222 self.pipe.unet.set_attn_processor(
223 processor=self.controlnet_attn_proc)
224 self.pipe.controlnet.set_attn_processor(
225 processor=self.controlnet_attn_proc)
226
227 added_prompt = 'best quality, extremely detailed'
228 negative_prompts = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'
229
230 video, fps = utils.prepare_video(
231 video_path, resolution, self.device, self.dtype, False)
232 control = utils.pre_process_depth(
233 video).to(self.device).to(self.dtype)
234
235 # depth_map_to_save = list(rearrange(control, 'f c w h -> f w h c').cpu().detach().numpy())
236 # _ = utils.create_video(depth_map_to_save, 4, path="ddxk.mp4", watermark=None)
237
238 f, _, h, w = video.shape
239 self.generator.manual_seed(seed)
240 latents = torch.randn((1, 4, h//8, w//8), dtype=self.dtype,
241 device=self.device, generator=self.generator)
242 latents = latents.repeat(f, 1, 1, 1)
243 result = self.inference(image=control,
244 prompt=prompt + ', ' + added_prompt,
245 height=h,
246 width=w,
247 negative_prompt=negative_prompts,
248 num_inference_steps=num_inference_steps,
249 guidance_scale=guidance_scale,
250 controlnet_conditioning_scale=controlnet_conditioning_scale,
251 eta=eta,
252 latents=latents,
253 seed=seed,
254 output_type='numpy',
255 split_to_chunks=True,

Callers

nothing calls this directly

Calls 2

set_modelMethod · 0.95
inferenceMethod · 0.95

Tested by

no test coverage detected