(self, d_model_size, num_heads, dff, rate=0.1)
| 169 | |
| 170 | class EncoderLayer(torch.nn.Module): |
| 171 | def __init__(self, d_model_size, num_heads, dff, rate=0.1): |
| 172 | super().__init__() |
| 173 | |
| 174 | self.multi_head_attention = MultiHeadAttention(d_model_size, num_heads) |
| 175 | self.ffn = point_wise_feed_forward_network(d_model_size, dff) |
| 176 | |
| 177 | self.layernorm1 = torch.nn.LayerNorm(d_model_size, eps=1e-6) |
| 178 | self.layernorm2 = torch.nn.LayerNorm(d_model_size, eps=1e-6) |
| 179 | |
| 180 | self.dropout1 = torch.nn.Dropout(rate) |
| 181 | self.dropout2 = torch.nn.Dropout(rate) |
| 182 | |
| 183 | def forward( |
| 184 | self, x, mask, layer_past=None, attention_mask=None, head_mask=None, use_cache=False, output_attentions=False |
nothing calls this directly
no test coverage detected