(
self,
hidden_size,
num_attention_heads,
layer_number,
layernorm_epsilon=1e-5,
)
| 484 | """ |
| 485 | |
| 486 | def __init__( |
| 487 | self, |
| 488 | hidden_size, |
| 489 | num_attention_heads, |
| 490 | layer_number, |
| 491 | layernorm_epsilon=1e-5, |
| 492 | ): |
| 493 | super(TopQueryLayer, self).__init__() |
| 494 | self.hidden_size = hidden_size |
| 495 | self.num_attention_heads = num_attention_heads |
| 496 | self.layernorm_epsilon = layernorm_epsilon |
| 497 | self.layer_number = layer_number |
| 498 | |
| 499 | # Use FP32 for Layernorm |
| 500 | self.input_layernorm = torch.nn.LayerNorm(self.hidden_size, |
| 501 | eps=self.layernorm_epsilon) |
| 502 | |
| 503 | # Self attention. |
| 504 | self.attention = TopQuerySelfAttention(self.hidden_size, |
| 505 | self.num_attention_heads, |
| 506 | self.layer_number) |
| 507 | # Layernorm on the input data. |
| 508 | self.post_attention_layernorm = torch.nn.LayerNorm(self.hidden_size, |
| 509 | eps=self.layernorm_epsilon) |
| 510 | |
| 511 | # MLP |
| 512 | self.mlp = MLP(self.hidden_size) |
| 513 | |
| 514 | def forward( |
| 515 | self, |
nothing calls this directly
no test coverage detected