| 13 | |
| 14 | |
| 15 | class VideoTransformerBlock(nn.Module): |
| 16 | ATTENTION_MODES = { |
| 17 | "softmax": CrossAttention, |
| 18 | "softmax-xformers": MemoryEfficientCrossAttention, |
| 19 | } |
| 20 | |
| 21 | def __init__( |
| 22 | self, |
| 23 | dim, |
| 24 | n_heads, |
| 25 | d_head, |
| 26 | dropout=0.0, |
| 27 | context_dim=None, |
| 28 | gated_ff=True, |
| 29 | checkpoint=True, |
| 30 | timesteps=None, |
| 31 | ff_in=False, |
| 32 | inner_dim=None, |
| 33 | attn_mode="softmax", |
| 34 | disable_self_attn=False, |
| 35 | disable_temporal_crossattention=False, |
| 36 | switch_temporal_ca_to_sa=False, |
| 37 | ): |
| 38 | super().__init__() |
| 39 | |
| 40 | attn_cls = self.ATTENTION_MODES[attn_mode] |
| 41 | |
| 42 | self.ff_in = ff_in or inner_dim is not None |
| 43 | if inner_dim is None: |
| 44 | inner_dim = dim |
| 45 | |
| 46 | assert int(n_heads * d_head) == inner_dim |
| 47 | |
| 48 | self.is_res = inner_dim == dim |
| 49 | |
| 50 | if self.ff_in: |
| 51 | self.norm_in = nn.LayerNorm(dim) |
| 52 | self.ff_in = FeedForward(dim, dim_out=inner_dim, dropout=dropout, glu=gated_ff) |
| 53 | |
| 54 | self.timesteps = timesteps |
| 55 | self.disable_self_attn = disable_self_attn |
| 56 | if self.disable_self_attn: |
| 57 | self.attn1 = attn_cls( |
| 58 | query_dim=inner_dim, |
| 59 | heads=n_heads, |
| 60 | dim_head=d_head, |
| 61 | context_dim=context_dim, |
| 62 | dropout=dropout, |
| 63 | ) # is a cross-attention |
| 64 | else: |
| 65 | self.attn1 = attn_cls( |
| 66 | query_dim=inner_dim, heads=n_heads, dim_head=d_head, dropout=dropout |
| 67 | ) # is a self-attention |
| 68 | |
| 69 | self.ff = FeedForward(inner_dim, dim_out=dim, dropout=dropout, glu=gated_ff) |
| 70 | |
| 71 | if disable_temporal_crossattention: |
| 72 | if switch_temporal_ca_to_sa: |