| 93 | |
| 94 | |
| 95 | class Encoder(nn.Module): |
| 96 | def __init__(self): |
| 97 | super(Encoder, self).__init__() |
| 98 | self.layer = nn.ModuleList() |
| 99 | self.encoder_norm = nn.LayerNorm(args['D'], eps=1e-6) |
| 100 | for _ in range(args['L']): |
| 101 | layer = Block() |
| 102 | self.layer.append(layer) |
| 103 | |
| 104 | def forward(self, x1, x2): |
| 105 | for layer_block in self.layer: |
| 106 | x1 = layer_block(x1, x2) |
| 107 | encoded = self.encoder_norm(x1) |
| 108 | return encoded |
| 109 | |
| 110 | |
| 111 | class Attention(nn.Module): |