(self, pos)
| 257 | return f"dim={self.freqs.shape[1] * 4}, n_heads={self.freqs.shape[0]}" |
| 258 | |
| 259 | def forward(self, pos): |
| 260 | theta_h = pos[..., None, 0:1] * self.freqs.to(pos.dtype) |
| 261 | theta_w = pos[..., None, 1:2] * self.freqs.to(pos.dtype) |
| 262 | return torch.cat((theta_h, theta_w), dim=-1) |
| 263 | |
| 264 | |
| 265 | # Shifted window attention |
nothing calls this directly
no outgoing calls
no test coverage detected