(self, dim, n_heads)
| 247 | |
| 248 | class AxialRoPE(nn.Module): |
| 249 | def __init__(self, dim, n_heads): |
| 250 | super().__init__() |
| 251 | log_min = math.log(math.pi) |
| 252 | log_max = math.log(10.0 * math.pi) |
| 253 | freqs = torch.linspace(log_min, log_max, n_heads * dim // 4 + 1)[:-1].exp() |
| 254 | self.register_buffer("freqs", freqs.view(dim // 4, n_heads).T.contiguous()) |
| 255 | |
| 256 | def extra_repr(self): |
| 257 | return f"dim={self.freqs.shape[1] * 4}, n_heads={self.freqs.shape[0]}" |