(
self,
hidden_size,
num_attention_heads,
layer_number,
layernorm_epsilon=1e-5,
)
| 577 | """ |
| 578 | |
| 579 | def __init__( |
| 580 | self, |
| 581 | hidden_size, |
| 582 | num_attention_heads, |
| 583 | layer_number, |
| 584 | layernorm_epsilon=1e-5, |
| 585 | ): |
| 586 | super(TopQueryLayer, self).__init__() |
| 587 | self.hidden_size = hidden_size |
| 588 | self.num_attention_heads = num_attention_heads |
| 589 | self.layernorm_epsilon = layernorm_epsilon |
| 590 | self.layer_number = layer_number |
| 591 | |
| 592 | # Use FP32 for Layernorm |
| 593 | self.input_layernorm = torch.nn.LayerNorm(self.hidden_size, |
| 594 | eps=self.layernorm_epsilon) |
| 595 | |
| 596 | # Self attention. |
| 597 | self.attention = TopQuerySelfAttention(self.hidden_size, |
| 598 | self.num_attention_heads, |
| 599 | self.layer_number) |
| 600 | # Layernorm on the input data. |
| 601 | self.post_attention_layernorm = torch.nn.LayerNorm(self.hidden_size, |
| 602 | eps=self.layernorm_epsilon) |
| 603 | |
| 604 | # MLP |
| 605 | self.mlp = MLP(self.hidden_size) |
| 606 | |
| 607 | def forward( |
| 608 | self, |
nothing calls this directly
no test coverage detected