Construct an EncoderLayer object.
(
self,
in_size,
size,
self_attn,
feed_forward,
dropout_rate,
normalize_before=True,
concat_after=False,
stochastic_depth_rate=0.0,
)
| 293 | |
| 294 | class EncoderLayerSANM(nn.Module): |
| 295 | def __init__( |
| 296 | self, |
| 297 | in_size, |
| 298 | size, |
| 299 | self_attn, |
| 300 | feed_forward, |
| 301 | dropout_rate, |
| 302 | normalize_before=True, |
| 303 | concat_after=False, |
| 304 | stochastic_depth_rate=0.0, |
| 305 | ): |
| 306 | """Construct an EncoderLayer object.""" |
| 307 | super(EncoderLayerSANM, self).__init__() |
| 308 | self.self_attn = self_attn |
| 309 | self.feed_forward = feed_forward |
| 310 | self.norm1 = LayerNorm(in_size) |
| 311 | self.norm2 = LayerNorm(size) |
| 312 | self.dropout = nn.Dropout(dropout_rate) |
| 313 | self.in_size = in_size |
| 314 | self.size = size |
| 315 | self.normalize_before = normalize_before |
| 316 | self.concat_after = concat_after |
| 317 | if self.concat_after: |
| 318 | self.concat_linear = nn.Linear(size + size, size) |
| 319 | self.stochastic_depth_rate = stochastic_depth_rate |
| 320 | self.dropout_rate = dropout_rate |
| 321 | |
| 322 | def forward(self, x, mask, cache=None, mask_shfit_chunk=None, mask_att_chunk_encoder=None): |
| 323 | """Compute encoded features. |