| 530 | return x |
| 531 | |
| 532 | class ShiftedWindowTransformerLayer(nn.Module): |
| 533 | def __init__(self, d_model, d_ff, d_head, cond_features, window_size, index, dropout=0.0): |
| 534 | super().__init__() |
| 535 | window_shift = window_size // 2 if index % 2 == 1 else 0 |
| 536 | self.self_attn = ShiftedWindowSelfAttentionBlock(d_model, d_head, cond_features, window_size, window_shift, dropout=dropout) |
| 537 | self.ff = FeedForwardBlock(d_model, d_ff, cond_features, dropout=dropout) |
| 538 | |
| 539 | def forward(self, x, pos, cond): |
| 540 | x = checkpoint(self.self_attn, x, pos, cond) |
| 541 | x = checkpoint(self.ff, x, cond) |
| 542 | return x |
| 543 | |
| 544 | |
| 545 | class NoAttentionTransformerLayer(nn.Module): |