| 202 | */ |
| 203 | template<class Float> |
| 204 | __device__ inline Float adam_update(Float parameter, Float gradient, Float &moment1, Float &moment2, |
| 205 | Float weight = 1) const { |
| 206 | Float regularized = weight * (gradient + weight_decay * parameter); |
| 207 | moment1 = beta1 * moment1 + (1 - beta1) * regularized; |
| 208 | moment2 = beta2 * moment2 + (1 - beta2) * regularized * regularized; |
| 209 | return lr * moment1 / (sqrt(moment2) + epsilon); |
| 210 | } |
| 211 | |
| 212 | protected: |
| 213 | Optimizer(const std::string &_type, int _num_moment = 0, float _lr = 0.025, float _weight_decay = 0, |
nothing calls this directly
no outgoing calls
no test coverage detected