| 519 | return x |
| 520 | |
| 521 | class NeighborhoodTransformerLayer(nn.Module): |
| 522 | def __init__(self, d_model, d_ff, d_head, cond_features, kernel_size, dropout=0.0): |
| 523 | super().__init__() |
| 524 | self.self_attn = NeighborhoodSelfAttentionBlock(d_model, d_head, cond_features, kernel_size, dropout=dropout) |
| 525 | self.ff = FeedForwardBlock(d_model, d_ff, cond_features, dropout=dropout) |
| 526 | |
| 527 | def forward(self, x, pos, cond): |
| 528 | x = checkpoint(self.self_attn, x, pos, cond) |
| 529 | x = checkpoint(self.ff, x, cond) |
| 530 | return x |
| 531 | |
| 532 | class ShiftedWindowTransformerLayer(nn.Module): |
| 533 | def __init__(self, d_model, d_ff, d_head, cond_features, window_size, index, dropout=0.0): |