(
x,
weight,
bias,
residual=None,
eps=1e-6,
prenorm=False,
residual_in_fp32=False
)
| 513 | |
| 514 | |
| 515 | def rms_norm_fn( |
| 516 | x, |
| 517 | weight, |
| 518 | bias, |
| 519 | residual=None, |
| 520 | eps=1e-6, |
| 521 | prenorm=False, |
| 522 | residual_in_fp32=False |
| 523 | ): |
| 524 | return LayerNormFn.apply( |
| 525 | x, |
| 526 | weight, |
| 527 | bias, |
| 528 | residual, |
| 529 | eps, |
| 530 | prenorm, |
| 531 | residual_in_fp32, |
| 532 | True |
| 533 | ) |
| 534 | |
| 535 | |
| 536 | class LayerNorm(nn.Module): |