(self, x, cache_x=None)
| 26 | self.padding = (0, 0, 0) |
| 27 | |
| 28 | def forward(self, x, cache_x=None): |
| 29 | padding = list(self._padding) |
| 30 | if cache_x is not None and self._padding[4] > 0: |
| 31 | cache_x = cache_x.to(x.device) |
| 32 | x = torch.cat([cache_x, x], dim=2) |
| 33 | padding[4] -= cache_x.shape[2] |
| 34 | x = F.pad(x, padding) |
| 35 | |
| 36 | return super().forward(x) |
| 37 | |
| 38 | |
| 39 | class RMS_norm(nn.Module): |