(
self,
hidden_size,
num_attention_heads,
layer_number,
layernorm_epsilon=1e-5,
fp16=True,
attention_softmax_in_fp32=True,
)
| 498 | """ |
| 499 | |
| 500 | def __init__( |
| 501 | self, |
| 502 | hidden_size, |
| 503 | num_attention_heads, |
| 504 | layer_number, |
| 505 | layernorm_epsilon=1e-5, |
| 506 | fp16=True, |
| 507 | attention_softmax_in_fp32=True, |
| 508 | ): |
| 509 | super(TransformerLayer, self).__init__() |
| 510 | self.hidden_size = hidden_size |
| 511 | self.layernorm_epsilon = layernorm_epsilon |
| 512 | self.layer_number = layer_number |
| 513 | |
| 514 | # Layernorm on the input data. |
| 515 | self.input_layernorm = torch.nn.LayerNorm(hidden_size, |
| 516 | eps=self.layernorm_epsilon) |
| 517 | |
| 518 | # Self attention. |
| 519 | self.attention = SelfAttention(hidden_size, |
| 520 | num_attention_heads, |
| 521 | layer_number, |
| 522 | fp16, |
| 523 | attention_softmax_in_fp32) |
| 524 | |
| 525 | # Layernorm on the input data. |
| 526 | self.post_attention_layernorm = torch.nn.LayerNorm(self.hidden_size, |
| 527 | eps=self.layernorm_epsilon) |
| 528 | self.mlp = MLP(self.hidden_size) |
| 529 | |
| 530 | def forward( |
| 531 | self, |
nothing calls this directly
no test coverage detected