| 28 | |
| 29 | |
| 30 | class EncoderLayer(nn.Module): |
| 31 | |
| 32 | def __init__(self, sa_block_cfg=None, ca_block_cfg=None, ffn_cfg=None): |
| 33 | super().__init__() |
| 34 | self.sa_block = build_attention(sa_block_cfg) |
| 35 | self.ffn = FFN(**ffn_cfg) |
| 36 | |
| 37 | def forward(self, **kwargs): |
| 38 | if self.sa_block is not None: |
| 39 | x = self.sa_block(**kwargs) |
| 40 | kwargs.update({'x': x}) |
| 41 | if self.ffn is not None: |
| 42 | x = self.ffn(**kwargs) |
| 43 | return x |
| 44 | |
| 45 | |
| 46 | class RetrievalDatabase(nn.Module): |