(self)
| 545 | self.init_weights() |
| 546 | |
| 547 | def init_freqs(self): |
| 548 | d = self.dim // self.num_heads |
| 549 | self.freqs = torch.cat([ |
| 550 | rope_params(1024, d - 4 * (d // 6)), |
| 551 | rope_params(1024, 2 * (d // 6)), |
| 552 | rope_params(1024, 2 * (d // 6)) |
| 553 | ], |
| 554 | dim=1) |
| 555 | |
| 556 | def teacache_init( |
| 557 | self, |