(self, activation)
| 136 | return layer |
| 137 | |
| 138 | def make_activation(self, activation): |
| 139 | if activation is None: |
| 140 | return nn.Identity() |
| 141 | if activation == "relu": |
| 142 | return nn.ReLU(inplace=True) |
| 143 | elif activation == "silu": |
| 144 | return nn.SiLU(inplace=True) |
| 145 | elif activation == "gelu": |
| 146 | return GELU_() |
| 147 | elif activation == "tanh": |
| 148 | return nn.Tanh() |
| 149 | else: |
| 150 | raise NotImplementedError |
| 151 | |
| 152 | class MultiHeadAttention(nn.Module): |
| 153 | """ |