| 149 | |
| 150 | |
| 151 | class VAEDecoderWrapperSingle(nn.Module): |
| 152 | def __init__(self): |
| 153 | super().__init__() |
| 154 | self.decoder = VAEDecoder3d() |
| 155 | mean = [ |
| 156 | -0.7571, -0.7089, -0.9113, 0.1075, -0.1745, 0.9653, -0.1517, 1.5508, |
| 157 | 0.4134, -0.0715, 0.5517, -0.3632, -0.1922, -0.9497, 0.2503, -0.2921 |
| 158 | ] |
| 159 | std = [ |
| 160 | 2.8184, 1.4541, 2.3275, 2.6558, 1.2196, 1.7708, 2.6052, 2.0743, |
| 161 | 3.2687, 2.1526, 2.8652, 1.5579, 1.6382, 1.1253, 2.8251, 1.9160 |
| 162 | ] |
| 163 | self.mean = torch.tensor(mean, dtype=torch.float32) |
| 164 | self.std = torch.tensor(std, dtype=torch.float32) |
| 165 | self.z_dim = 16 |
| 166 | self.conv2 = CausalConv3d(self.z_dim, self.z_dim, 1) |
| 167 | |
| 168 | def forward( |
| 169 | self, |
| 170 | z: torch.Tensor, |
| 171 | is_first_frame: torch.Tensor, |
| 172 | *feat_cache: List[torch.Tensor] |
| 173 | ): |
| 174 | # from [batch_size, num_frames, num_channels, height, width] |
| 175 | # to [batch_size, num_channels, num_frames, height, width] |
| 176 | z = z.permute(0, 2, 1, 3, 4) |
| 177 | assert z.shape[2] == 1 |
| 178 | feat_cache = list(feat_cache) |
| 179 | is_first_frame = is_first_frame.bool() |
| 180 | |
| 181 | device, dtype = z.device, z.dtype |
| 182 | scale = [self.mean.to(device=device, dtype=dtype), |
| 183 | 1.0 / self.std.to(device=device, dtype=dtype)] |
| 184 | |
| 185 | if isinstance(scale[0], torch.Tensor): |
| 186 | z = z / scale[1].view(1, self.z_dim, 1, 1, 1) + scale[0].view( |
| 187 | 1, self.z_dim, 1, 1, 1) |
| 188 | else: |
| 189 | z = z / scale[1] + scale[0] |
| 190 | x = self.conv2(z) |
| 191 | out, feat_cache = self.decoder(x, is_first_frame, feat_cache=feat_cache) |
| 192 | out = out.clamp_(-1, 1) |
| 193 | # from [batch_size, num_channels, num_frames, height, width] |
| 194 | # to [batch_size, num_frames, num_channels, height, width] |
| 195 | out = out.permute(0, 2, 1, 3, 4) |
| 196 | return out, feat_cache |
| 197 | |
| 198 | |
| 199 | class VAEDecoder3d(nn.Module): |