| 13 | |
| 14 | |
| 15 | class SparseResBlock3d(nn.Module): |
| 16 | def __init__( |
| 17 | self, |
| 18 | channels: int, |
| 19 | emb_channels: int, |
| 20 | out_channels: Optional[int] = None, |
| 21 | downsample: bool = False, |
| 22 | upsample: bool = False, |
| 23 | ): |
| 24 | super().__init__() |
| 25 | self.channels = channels |
| 26 | self.emb_channels = emb_channels |
| 27 | self.out_channels = out_channels or channels |
| 28 | self.downsample = downsample |
| 29 | self.upsample = upsample |
| 30 | |
| 31 | assert not (downsample and upsample), "Cannot downsample and upsample at the same time" |
| 32 | |
| 33 | self.norm1 = LayerNorm32(channels, elementwise_affine=True, eps=1e-6) |
| 34 | self.norm2 = LayerNorm32(self.out_channels, elementwise_affine=False, eps=1e-6) |
| 35 | self.conv1 = sp.SparseConv3d(channels, self.out_channels, 3) |
| 36 | self.conv2 = zero_module(sp.SparseConv3d(self.out_channels, self.out_channels, 3)) |
| 37 | self.emb_layers = nn.Sequential( |
| 38 | nn.SiLU(), |
| 39 | nn.Linear(emb_channels, 2 * self.out_channels, bias=True), |
| 40 | ) |
| 41 | self.skip_connection = sp.SparseLinear(channels, self.out_channels) if channels != self.out_channels else nn.Identity() |
| 42 | self.updown = None |
| 43 | if self.downsample: |
| 44 | self.updown = sp.SparseDownsample(2) |
| 45 | elif self.upsample: |
| 46 | self.updown = sp.SparseUpsample(2) |
| 47 | |
| 48 | def _updown(self, x: sp.SparseTensor) -> sp.SparseTensor: |
| 49 | if self.updown is not None: |
| 50 | x = self.updown(x) |
| 51 | return x |
| 52 | |
| 53 | def forward(self, x: sp.SparseTensor, emb: torch.Tensor) -> sp.SparseTensor: |
| 54 | emb_out = self.emb_layers(emb).type(x.dtype) |
| 55 | scale, shift = torch.chunk(emb_out, 2, dim=1) |
| 56 | |
| 57 | x = self._updown(x) |
| 58 | h = x.replace(self.norm1(x.feats)) |
| 59 | h = h.replace(F.silu(h.feats)) |
| 60 | h = self.conv1(h) |
| 61 | h = h.replace(self.norm2(h.feats)) * (1 + scale) + shift |
| 62 | h = h.replace(F.silu(h.feats)) |
| 63 | h = self.conv2(h) |
| 64 | h = h + self.skip_connection(x) |
| 65 | |
| 66 | return h |
| 67 | |
| 68 | |
| 69 | class SLatFlowModel(nn.Module): |