(self)
| 546 | self.init_weights() |
| 547 | |
| 548 | def init_freqs(self): |
| 549 | d = self.dim // self.num_heads |
| 550 | self.freqs = torch.cat([ |
| 551 | rope_params(1024, d - 4 * (d // 6)), |
| 552 | rope_params(1024, 2 * (d // 6)), |
| 553 | rope_params(1024, 2 * (d // 6)) |
| 554 | ], |
| 555 | dim=1) |
| 556 | |
| 557 | def teacache_init( |
| 558 | self, |