history += grad*grad; value = value - lr*grad/sqrt(history+delta)
| 26 | // history += grad*grad; |
| 27 | // value = value - lr*grad/sqrt(history+delta) |
| 28 | void AdaGrad::Apply(int epoch, float lr, const string& name, |
| 29 | Tensor& grad, Tensor& value, int step) { |
| 30 | if (grad.empty()) |
| 31 | return; |
| 32 | ApplyRegularizerConstraint(epoch, name, value, grad, step); |
| 33 | if (learning_rate_multplier_.find(name) != learning_rate_multplier_.end()) |
| 34 | lr *= learning_rate_multplier_.at(name); |
| 35 | |
| 36 | if (history_gradient_.find(name) == history_gradient_.end()) { |
| 37 | history_gradient_[name].ResetLike(value); |
| 38 | history_gradient_[name].SetValue(0.0f); |
| 39 | } |
| 40 | Tensor& history = history_gradient_[name]; |
| 41 | Tensor tmp = Square(grad); |
| 42 | history += tmp; |
| 43 | Add(history, delta_, &tmp); |
| 44 | Sqrt(tmp, &tmp); |
| 45 | Div(grad, tmp, &tmp); |
| 46 | Axpy(-lr, tmp, &value); |
| 47 | } |
| 48 | } // namespace singa |
| 49 | #endif // SRC_MODEL_OPTIMIZER_ADAGRAD_H_ |