| 288 | |
| 289 | |
| 290 | class AttentionModule(nn.Module): |
| 291 | def __init__(self, num_heads): |
| 292 | super().__init__() |
| 293 | self.num_heads = num_heads |
| 294 | |
| 295 | def forward(self, q, k, v, attention_mask=None): |
| 296 | x = flash_attention(q=q, k=k, v=v, num_heads=self.num_heads, attention_mask=attention_mask) |
| 297 | return x |
| 298 | |
| 299 | |
| 300 | class SelfAttention(nn.Module): |