| 56 | |
| 57 | |
| 58 | class SGD(OptimizerBase): |
| 59 | def __init__( |
| 60 | self, lr=0.01, momentum=0.0, clip_norm=None, lr_scheduler=None, **kwargs |
| 61 | ): |
| 62 | """ |
| 63 | A stochastic gradient descent optimizer. |
| 64 | |
| 65 | Notes |
| 66 | ----- |
| 67 | For model parameters :math:`\\theta`, averaged parameter gradients |
| 68 | :math:`\\nabla_{\\theta} \mathcal{L}`, and learning rate :math:`\eta`, |
| 69 | the SGD update at timestep `t` is |
| 70 | |
| 71 | .. math:: |
| 72 | |
| 73 | \\text{update}^{(t)} |
| 74 | &= \\text{momentum} \cdot \\text{update}^{(t-1)} + \eta^{(t)} \\nabla_{\\theta} \mathcal{L}\\\\ |
| 75 | \\theta^{(t+1)} |
| 76 | &\leftarrow \\theta^{(t)} - \\text{update}^{(t)} |
| 77 | |
| 78 | Parameters |
| 79 | ---------- |
| 80 | lr : float |
| 81 | Learning rate for SGD. If scheduler is not None, this is used as |
| 82 | the starting learning rate. Default is 0.01. |
| 83 | momentum : float in range [0, 1] |
| 84 | The fraction of the previous update to add to the current update. |
| 85 | If 0, no momentum is applied. Default is 0. |
| 86 | clip_norm : float |
| 87 | If not None, all param gradients are scaled to have maximum l2 norm of |
| 88 | `clip_norm` before computing update. Default is None. |
| 89 | lr_scheduler : str, :doc:`Scheduler <numpy_ml.neural_nets.schedulers>` object, or None |
| 90 | The learning rate scheduler. If None, use a constant learning |
| 91 | rate equal to `lr`. Default is None. |
| 92 | """ |
| 93 | super().__init__(lr, lr_scheduler) |
| 94 | |
| 95 | self.hyperparameters = { |
| 96 | "id": "SGD", |
| 97 | "lr": lr, |
| 98 | "momentum": momentum, |
| 99 | "clip_norm": clip_norm, |
| 100 | "lr_scheduler": str(self.lr_scheduler), |
| 101 | } |
| 102 | |
| 103 | def __str__(self): |
| 104 | H = self.hyperparameters |
| 105 | lr, mm, cn, sc = H["lr"], H["momentum"], H["clip_norm"], H["lr_scheduler"] |
| 106 | return "SGD(lr={}, momentum={}, clip_norm={}, lr_scheduler={})".format( |
| 107 | lr, mm, cn, sc |
| 108 | ) |
| 109 | |
| 110 | def update(self, param, param_grad, param_name, cur_loss=None): |
| 111 | """ |
| 112 | Compute the SGD update for a given parameter |
| 113 | |
| 114 | Parameters |
| 115 | ---------- |
no outgoing calls
no test coverage detected