AdaLayerNormZero.forward with LoRA applied to the inner ``linear``.
(norm, x, emb, adapter)
| 185 | |
| 186 | |
| 187 | def _adanorm_zero_forward(norm, x, emb, adapter): |
| 188 | """AdaLayerNormZero.forward with LoRA applied to the inner ``linear``.""" |
| 189 | emb = lora_forward(norm.linear, norm.silu(emb), adapter) |
| 190 | shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = emb.chunk(6, dim=1) |
| 191 | x = norm.norm(x) * (1 + scale_msa[:, None]) + shift_msa[:, None] |
| 192 | return x, gate_msa, shift_mlp, scale_mlp, gate_mlp |
| 193 | |
| 194 | |
| 195 | def _adanorm_zero_single_forward(norm, x, emb, adapter): |
no test coverage detected