| 147 | m.bias.grad.data.mul_(nu) |
| 148 | |
| 149 | def _step(self, closure): |
| 150 | # FIXME (CW): Modified based on SGD (removed nestrov and dampening in momentum.) |
| 151 | # FIXME (CW): 1. no nesterov, 2. buf.mul_(momentum).add_(1 <del> - dampening </del>, d_p) |
| 152 | for group in self.param_groups: |
| 153 | weight_decay = group['weight_decay'] |
| 154 | momentum = group['momentum'] |
| 155 | |
| 156 | for p in group['params']: |
| 157 | if p.grad is None: |
| 158 | continue |
| 159 | d_p = p.grad.data |
| 160 | if weight_decay != 0 and self.steps >= 20 * self.TCov: |
| 161 | d_p.add_(p.data, alpha=weight_decay) |
| 162 | if momentum != 0: |
| 163 | param_state = self.state[p] |
| 164 | if 'momentum_buffer' not in param_state: |
| 165 | buf = param_state['momentum_buffer'] = torch.zeros_like(p.data) |
| 166 | buf.mul_(momentum).add_(d_p) |
| 167 | else: |
| 168 | buf = param_state['momentum_buffer'] |
| 169 | buf.mul_(momentum).add_(d_p, alpha=1) |
| 170 | d_p = buf |
| 171 | |
| 172 | p.data.add_(d_p, alpha=-group['lr']) |
| 173 | |
| 174 | def step(self, closure=None): |
| 175 | # FIXME(CW): temporal fix for compatibility with Official LR scheduler. |