(self, init_std: float)
| 267 | return self.w2(F.silu(self.w1(x)) * self.w3(x)) |
| 268 | |
| 269 | def init_weights(self, init_std: float): |
| 270 | nn.init.trunc_normal_(self.w1.weight, mean=0.0, std=0.02) |
| 271 | for linear in (self.w2, self.w3): |
| 272 | nn.init.trunc_normal_(linear.weight, mean=0.0, std=init_std) |
| 273 | |
| 274 | |
| 275 | class TransformerBlock(nn.Module): |
nothing calls this directly
no outgoing calls
no test coverage detected