| 24 | |
| 25 | class DynamicLinear(nn.Module): |
| 26 | def __init__(self, in_features, out_features, bias=True): |
| 27 | super().__init__() |
| 28 | self.in_features = in_features |
| 29 | self.out_features = out_features |
| 30 | |
| 31 | self.weight = nn.Parameter(torch.randn(out_features, in_features)) |
| 32 | if bias: |
| 33 | self.bias = nn.Parameter(torch.zeros(out_features)) |
| 34 | else: |
| 35 | self.register_parameter('bias', None) |
| 36 | |
| 37 | nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5)) |
| 38 | if self.bias is not None: |
| 39 | fan_in, _ = nn.init._calculate_fan_in_and_fan_out(self.weight) |
| 40 | bound = 1 / math.sqrt(fan_in) |
| 41 | nn.init.uniform_(self.bias, -bound, bound) |
| 42 | |
| 43 | def forward(self, x): |
| 44 | return F.linear(x, self.weight, self.bias) |