| 156 | |
| 157 | |
| 158 | class RMSprop(Optimizer): |
| 159 | def __init__(self, learning_rate=0.001, rho=0.9, epsilon=1e-8): |
| 160 | self.eps = epsilon |
| 161 | self.rho = rho |
| 162 | self.lr = learning_rate |
| 163 | |
| 164 | def update(self, network): |
| 165 | for i, layer in enumerate(network.parametric_layers): |
| 166 | for n in layer.parameters.keys(): |
| 167 | grad = layer.parameters.grad[n] |
| 168 | self.accu[i][n] = (self.rho * self.accu[i][n]) + (1.0 - self.rho) * ( |
| 169 | grad**2 |
| 170 | ) |
| 171 | step = self.lr * grad / (np.sqrt(self.accu[i][n]) + self.eps) |
| 172 | layer.parameters.step(n, -step) |
| 173 | |
| 174 | def setup(self, network): |
| 175 | # Accumulators |
| 176 | self.accu = defaultdict(dict) |
| 177 | for i, layer in enumerate(network.parametric_layers): |
| 178 | for n in layer.parameters.keys(): |
| 179 | self.accu[i][n] = np.zeros_like(layer.parameters[n]) |
| 180 | |
| 181 | |
| 182 | class Adam(Optimizer): |
no outgoing calls