(self, in_features, out_features, act='gelu', bias=True)
| 154 | __constants__ = ['bias'] |
| 155 | |
| 156 | def __init__(self, in_features, out_features, act='gelu', bias=True): |
| 157 | super(LinearActivation, self).__init__() |
| 158 | self.in_features = in_features |
| 159 | self.out_features = out_features |
| 160 | self.fused_gelu = False |
| 161 | self.fused_tanh = False |
| 162 | if isinstance(act, str) or (sys.version_info[0] == 2 and isinstance(act, unicode)): |
| 163 | if bias and act == 'gelu': |
| 164 | self.fused_gelu = True |
| 165 | elif bias and act == 'tanh': |
| 166 | self.fused_tanh = True |
| 167 | else: |
| 168 | self.act_fn = ACT2FN[act] |
| 169 | else: |
| 170 | self.act_fn = act |
| 171 | self.weight = Parameter(torch.Tensor(out_features, in_features)) |
| 172 | if bias: |
| 173 | self.bias = Parameter(torch.Tensor(out_features)) |
| 174 | else: |
| 175 | self.register_parameter('bias', None) |
| 176 | self.reset_parameters() |
| 177 | |
| 178 | def reset_parameters(self): |
| 179 | init.kaiming_uniform_(self.weight, a=math.sqrt(5)) |
nothing calls this directly
no test coverage detected