(self, network)
| 162 | self.lr = learning_rate |
| 163 | |
| 164 | def update(self, network): |
| 165 | for i, layer in enumerate(network.parametric_layers): |
| 166 | for n in layer.parameters.keys(): |
| 167 | grad = layer.parameters.grad[n] |
| 168 | self.accu[i][n] = (self.rho * self.accu[i][n]) + (1.0 - self.rho) * ( |
| 169 | grad**2 |
| 170 | ) |
| 171 | step = self.lr * grad / (np.sqrt(self.accu[i][n]) + self.eps) |
| 172 | layer.parameters.step(n, -step) |
| 173 | |
| 174 | def setup(self, network): |
| 175 | # Accumulators |