Method __call__

diffsynth/pipelines/svd_video.py:111–188

(
        self,
        input_image=None,
        input_video=None,
        mask_frames=[],
        mask_frame_ids=[],
        min_cfg_scale=1.0,
        max_cfg_scale=3.0,
        denoising_strength=1.0,
        num_frames=25,
        height=576,
        width=1024,
        fps=7,
        motion_bucket_id=127,
        noise_aug_strength=0.02,
        num_inference_steps=20,
        post_normalize=True,
        contrast_enhance_scale=1.2,
        seed=None,
        progress_bar_cmd=tqdm,
        progress_bar_st=None,
    )

    @torch.no_grad()
    def __call__(
        self,
        input_image=None,
        input_video=None,
        mask_frames=[],
        mask_frame_ids=[],
        min_cfg_scale=1.0,
        max_cfg_scale=3.0,
        denoising_strength=1.0,
        num_frames=25,
        height=576,
        width=1024,
        fps=7,
        motion_bucket_id=127,
        noise_aug_strength=0.02,
        num_inference_steps=20,
        post_normalize=True,
        contrast_enhance_scale=1.2,
        seed=None,
        progress_bar_cmd=tqdm,
        progress_bar_st=None,
    ):
        height, width = self.check_resize_height_width(height, width)
        
        # Prepare scheduler
        self.scheduler.set_timesteps(num_inference_steps, denoising_strength=denoising_strength)

        # Prepare latent tensors
        noise = self.generate_noise((num_frames, 4, height//8, width//8), seed=seed, device=self.device, dtype=self.torch_dtype)
        if denoising_strength == 1.0:
            latents = noise.clone()
        else:
            latents = self.encode_video_with_vae(input_video)
            latents = self.scheduler.add_noise(latents, noise, self.scheduler.timesteps[0])

        # Prepare mask frames
        if len(mask_frames) > 0:
            mask_latents = self.encode_video_with_vae(mask_frames)

        # Encode image
        image_emb_clip_posi = self.encode_image_with_clip(input_image)
        image_emb_clip_nega = torch.zeros_like(image_emb_clip_posi)
        image_emb_vae_posi = repeat(self.encode_image_with_vae(input_image, noise_aug_strength, seed=seed), "B C H W -> (B T) C H W", T=num_frames)
        image_emb_vae_nega = torch.zeros_like(image_emb_vae_posi)

        # Prepare classifier-free guidance
        cfg_scales = torch.linspace(min_cfg_scale, max_cfg_scale, num_frames)
        cfg_scales = cfg_scales.reshape(num_frames, 1, 1, 1).to(device=self.device, dtype=self.torch_dtype)
        
        # Prepare positional id
        add_time_id = torch.tensor([[fps-1, motion_bucket_id, noise_aug_strength]], device=self.device)

        # Denoise
        for progress_id, timestep in enumerate(progress_bar_cmd(self.scheduler.timesteps)):

            # Mask frames
            for frame_id, mask_frame_id in enumerate(mask_frame_ids):
                latents[mask_frame_id] = self.scheduler.add_noise(mask_latents[frame_id], noise[mask_frame_id], timestep)