residual_in_fp32 only has an effect if residual is None. Otherwise residual dtype is residual.dtype.
(
x0,
residual,
weight,
bias,
dropout_p,
epsilon,
rowscale=None,
layerscale=None,
prenorm=False,
residual_in_fp32=False,
return_dropout_mask=False,
)
| 659 | |
| 660 | |
| 661 | def dropout_add_layer_norm( |
| 662 | x0, |
| 663 | residual, |
| 664 | weight, |
| 665 | bias, |
| 666 | dropout_p, |
| 667 | epsilon, |
| 668 | rowscale=None, |
| 669 | layerscale=None, |
| 670 | prenorm=False, |
| 671 | residual_in_fp32=False, |
| 672 | return_dropout_mask=False, |
| 673 | ): |
| 674 | """residual_in_fp32 only has an effect if residual is None. |
| 675 | Otherwise residual dtype is residual.dtype. |
| 676 | """ |
| 677 | return DropoutAddLayerNormFn.apply( |
| 678 | x0, |
| 679 | residual, |
| 680 | weight, |
| 681 | bias, |
| 682 | rowscale, |
| 683 | layerscale, |
| 684 | dropout_p, |
| 685 | epsilon, |
| 686 | residual_in_fp32, |
| 687 | prenorm, |
| 688 | False, |
| 689 | return_dropout_mask, |
| 690 | ) |
| 691 | |
| 692 | |
| 693 | def dropout_add_layer_norm_subset( |