| 69 | |
| 70 | |
| 71 | class SGD(Optimizer): |
| 72 | def __init__(self, learning_rate=0.01, momentum=0.9, decay=0.0, nesterov=False): |
| 73 | self.nesterov = nesterov |
| 74 | self.decay = decay |
| 75 | self.momentum = momentum |
| 76 | self.lr = learning_rate |
| 77 | self.iteration = 0 |
| 78 | self.velocity = None |
| 79 | |
| 80 | def update(self, network): |
| 81 | lr = self.lr * (1.0 / (1.0 + self.decay * self.iteration)) |
| 82 | |
| 83 | for i, layer in enumerate(network.parametric_layers): |
| 84 | for n in layer.parameters.keys(): |
| 85 | # Get gradient values |
| 86 | grad = layer.parameters.grad[n] |
| 87 | update = self.momentum * self.velocity[i][n] - lr * grad |
| 88 | self.velocity[i][n] = update |
| 89 | if self.nesterov: |
| 90 | # Adjust using updated velocity |
| 91 | update = self.momentum * self.velocity[i][n] - lr * grad |
| 92 | layer.parameters.step(n, update) |
| 93 | self.iteration += 1 |
| 94 | |
| 95 | def setup(self, network): |
| 96 | self.velocity = defaultdict(dict) |
| 97 | for i, layer in enumerate(network.parametric_layers): |
| 98 | for n in layer.parameters.keys(): |
| 99 | self.velocity[i][n] = np.zeros_like(layer.parameters[n]) |
| 100 | |
| 101 | |
| 102 | class Adagrad(Optimizer): |