(x)
| 33 | |
| 34 | |
| 35 | def nonlinearity(x): |
| 36 | # this incurs additional memory: return x*torch.sigmoid(x) |
| 37 | # so we switch to pytorch's in-built function and perform silu in-place: |
| 38 | return nn.SiLU(inplace=True)(x) |
| 39 | |
| 40 | |
| 41 | def Normalize(in_channels, num_groups=32): |