| 40 | |
| 41 | class CrossAttention(nn.Module): |
| 42 | def __init__(self, embed_dim, num_heads, dropout=0.1): |
| 43 | super().__init__() |
| 44 | self.embed_dim = embed_dim |
| 45 | self.num_heads = num_heads |
| 46 | self.head_dim = embed_dim // num_heads |
| 47 | self.scale = self.head_dim ** -0.5 |
| 48 | |
| 49 | self.q_proj = nn.Linear(embed_dim, embed_dim) |
| 50 | self.k_proj = nn.Linear(embed_dim, embed_dim) |
| 51 | self.v_proj = nn.Linear(embed_dim, embed_dim) |
| 52 | self.out_proj = nn.Linear(embed_dim, embed_dim) |
| 53 | self.dropout = nn.Dropout(dropout) |
| 54 | |
| 55 | def forward(self, query, key, value, mask=None): |
| 56 | B, N_q, C = query.shape |