| 26 | |
| 27 | |
| 28 | class StaticEncoder(nn.Module): |
| 29 | def __init__(self, opt: Options, **model_kwargs): |
| 30 | super(StaticEncoder, self).__init__() |
| 31 | self.opt = opt |
| 32 | self.model = GSPredictor(opt, **model_kwargs) |
| 33 | if hasattr(opt, 'compile') and opt.compile: |
| 34 | self.model = torch.compile(self.model) |
| 35 | self.gaussian_renderer = gaussian_renderer_dynamic.render |
| 36 | self.background = torch.tensor(opt.background_color, dtype=torch.float32, device="cuda") |
| 37 | |
| 38 | # LPIPS loss |
| 39 | self.lpips_loss = LPIPS(net='vgg') |
| 40 | self.lpips_loss.eval() |
| 41 | self.lpips_loss.requires_grad_(False) |
| 42 | |
| 43 | def state_dict(self, **kwargs): |
| 44 | # remove lpips_loss |
| 45 | state_dict = super().state_dict(**kwargs) |
| 46 | for k in list(state_dict.keys()): |
| 47 | if 'lpips_loss' in k: |
| 48 | del state_dict[k] |
| 49 | return state_dict |
| 50 | |
| 51 | def load_state_dict(self, state_dict, strict=True): |
| 52 | # Optionally, if the LPIPS keys exist in the state_dict, remove them before loading. |
| 53 | filtered_state_dict = {k: v for k, v in state_dict.items() if not k.startswith('lpips_loss')} |
| 54 | return super().load_state_dict(filtered_state_dict, strict=False) |
| 55 | |
| 56 | def train(self, mode=True): |
| 57 | super().train(mode) |
| 58 | if 'lpips_loss' in self.__dict__: |
| 59 | self.lpips_loss.eval() |
| 60 | return self |
| 61 | |
| 62 | def forward_gaussians(self, frames, depths, cond_times=None): |
| 63 | # frames: [B, V, C, H, W] |
| 64 | # return: gaussians: [B, N, D] |
| 65 | decoder_out = self.model(frames, depths, cond_times=cond_times) |
| 66 | return decoder_out |
| 67 | |
| 68 | def forward(self, data, step_ratio=0.0): |
| 69 | # data: [B, 2, C, H, W] |
| 70 | input_frames = data['frames'][:, 0:1] # [B, 1, C, H, W], input features |
| 71 | input_depths = data['depths'][:, 0:1] # [B, 1, C, H, W], input features |
| 72 | |
| 73 | results = {} |
| 74 | |
| 75 | # predict gaussians |
| 76 | decoder_out = self.forward_gaussians(input_frames, input_depths) |
| 77 | with autocast('cuda', enabled=False): |
| 78 | render_pkg = self.gaussian_renderer(decoder_out["pred_gs"], self.background, opt=self.opt) |
| 79 | output_frames = render_pkg["render"] |
| 80 | pred_depths = render_pkg["depth"] |
| 81 | mse_loss = F.mse_loss(output_frames, input_frames) |
| 82 | loss = mse_loss |
| 83 | |
| 84 | if self.opt.depth_downsample: |
| 85 | actual_h = int(self.opt.down_resolution[0] * (2 ** self.opt.decoder_ratio / self.opt.patch_size)) |