(self)
| 362 | |
| 363 | class CLIPAttention: |
| 364 | def __init__(self): |
| 365 | self.embed_dim = 768 |
| 366 | self.num_heads = 12 |
| 367 | self.head_dim = self.embed_dim // self.num_heads |
| 368 | self.scale = self.head_dim**-0.5 |
| 369 | self.k_proj = Linear(self.embed_dim, self.embed_dim) |
| 370 | self.v_proj = Linear(self.embed_dim, self.embed_dim) |
| 371 | self.q_proj = Linear(self.embed_dim, self.embed_dim) |
| 372 | self.out_proj = Linear(self.embed_dim, self.embed_dim) |
| 373 | |
| 374 | def _shape(self, tensor, seq_len: int, bsz: int): |
| 375 | return tensor.reshape(bsz, seq_len, self.num_heads, self.head_dim).permute(0,2,1,3) |
nothing calls this directly
no outgoing calls
no test coverage detected