()
| 704 | torch.nn.init.normal_(self.position_embeddings.weight, mean=0.0, std=init_method_std) |
| 705 | |
| 706 | def get_layer(): |
| 707 | if use_decoder_layer: |
| 708 | return ParallelDecoderLayer( |
| 709 | hidden_size, |
| 710 | num_attention_heads, |
| 711 | attention_dropout_prob, |
| 712 | output_dropout_prob, |
| 713 | layernorm_epsilon, |
| 714 | unscaled_init_method(init_method_std), |
| 715 | output_layer_init_method=output_layer_init_method |
| 716 | ) |
| 717 | else: |
| 718 | return ParallelTransformerLayer( |
| 719 | hidden_size, |
| 720 | num_attention_heads, |
| 721 | attention_dropout_prob, |
| 722 | output_dropout_prob, |
| 723 | layernorm_epsilon, |
| 724 | unscaled_init_method(init_method_std), |
| 725 | output_layer_init_method=output_layer_init_method, |
| 726 | relative_encoding=relative_encoding, |
| 727 | performer=performer, |
| 728 | attention_scale=attention_scale) |
| 729 | |
| 730 | # Transformer layers. |
| 731 | self.layers = torch.nn.ModuleList( |
nothing calls this directly
no test coverage detected