history = history * rho + grad * grad * (1 - rho) value = value - lr * grad / sqrt(history + delta)
| 29 | // history = history * rho + grad * grad * (1 - rho) |
| 30 | // value = value - lr * grad / sqrt(history + delta) |
| 31 | void RMSProp::Apply(int epoch, float lr, const string& name, Tensor& grad, |
| 32 | Tensor& value, int step) { |
| 33 | if (grad.empty()) |
| 34 | return; |
| 35 | ApplyRegularizerConstraint(epoch, name, value, grad, step); |
| 36 | if (learning_rate_multplier_.find(name) != learning_rate_multplier_.end()) |
| 37 | lr *= learning_rate_multplier_.at(name); |
| 38 | |
| 39 | if (history_gradient_.find(name) == history_gradient_.end()) { |
| 40 | history_gradient_[name].ResetLike(value); |
| 41 | history_gradient_[name].SetValue(0.0f); |
| 42 | } |
| 43 | Tensor& history = history_gradient_[name]; |
| 44 | history *= rho_; |
| 45 | Tensor tmp = Square(grad); |
| 46 | Axpy(1 - rho_, tmp, &history); |
| 47 | Sqrt(history + delta_, &tmp); |
| 48 | Div(grad, tmp, &tmp); |
| 49 | Axpy(-lr, tmp, &value); |
| 50 | } |
| 51 | } // namespace singa |
| 52 | #endif // SRC_MODEL_OPTIMIZER_ADAGRAD_H_ |