tmp = history; history = lr * grad + history * mom tmp = (1+mom) * history - tmp * mom; value = value - tmp;
| 31 | // tmp = (1+mom) * history - tmp * mom; |
| 32 | // value = value - tmp; |
| 33 | void Nesterov::Apply(int epoch, float lr, const string& name, Tensor& grad, |
| 34 | Tensor& value, int step) { |
| 35 | if (grad.empty()) |
| 36 | return; |
| 37 | ApplyRegularizerConstraint(epoch, name, value, grad, step); |
| 38 | if (learning_rate_multplier_.find(name) != learning_rate_multplier_.end()) |
| 39 | lr *= learning_rate_multplier_.at(name); |
| 40 | if (momentum_generator_) { |
| 41 | float mom = momentum_generator_(step); |
| 42 | if (history_gradient_.find(name) == history_gradient_.end()) { |
| 43 | history_gradient_[name].ResetLike(value); |
| 44 | history_gradient_[name].SetValue(0.0f); |
| 45 | } |
| 46 | Tensor& history = history_gradient_[name]; |
| 47 | Tensor tmp = history.Clone(); |
| 48 | history *= mom; |
| 49 | Axpy(lr, grad, &history); |
| 50 | tmp *= -mom; |
| 51 | Axpy(1 + mom, history, &tmp); |
| 52 | value -= tmp; |
| 53 | } |
| 54 | } |
| 55 | } // namespace singa |
| 56 | #endif // SRC_MODEL_OPTIMIZER_NESTEROV_H_ |