(self, hidden_states)
| 91 | ) |
| 92 | |
| 93 | def forward(self, hidden_states): |
| 94 | # [s, b, 4hp] |
| 95 | intermediate_parallel, _ = self.dense_h_to_4h(hidden_states) |
| 96 | intermediate_parallel = self.activation_func(intermediate_parallel) |
| 97 | # [s, b, h] |
| 98 | output, output_bias = self.dense_4h_to_h(intermediate_parallel) |
| 99 | |
| 100 | return output, output_bias |
| 101 | |
| 102 | |
| 103 | class ParallelSelfAttention(MegatronModule): |