Causal 3d convolusion.
| 15 | |
| 16 | |
| 17 | class CausalConv3d(nn.Conv3d): |
| 18 | """ |
| 19 | Causal 3d convolusion. |
| 20 | """ |
| 21 | |
| 22 | def __init__(self, *args, **kwargs): |
| 23 | super().__init__(*args, **kwargs) |
| 24 | self._padding = (self.padding[2], self.padding[2], self.padding[1], |
| 25 | self.padding[1], 2 * self.padding[0], 0) |
| 26 | self.padding = (0, 0, 0) |
| 27 | |
| 28 | def forward(self, x, cache_x=None): |
| 29 | padding = list(self._padding) |
| 30 | if cache_x is not None and self._padding[4] > 0: |
| 31 | cache_x = cache_x.to(x.device) |
| 32 | x = torch.cat([cache_x, x], dim=2) |
| 33 | padding[4] -= cache_x.shape[2] |
| 34 | x = F.pad(x, padding) |
| 35 | |
| 36 | return super().forward(x) |
| 37 | |
| 38 | |
| 39 | class RMS_norm(nn.Module): |