| 41 | } |
| 42 | |
| 43 | SymbolVar SGD::make( |
| 44 | SymbolVar symbol_weight, SymbolVar symbol_grad, |
| 45 | std::shared_ptr<cg::ComputingGraph> graph) { |
| 46 | SymbolVar symbol_pre_grad; |
| 47 | auto pre_grad = TensorGen::zeros<dtype::Float32>( |
| 48 | symbol_grad.shape(), symbol_grad.node()->comp_node()); |
| 49 | m_pre_grads.push_back(pre_grad); |
| 50 | symbol_pre_grad = opr::SharedDeviceTensor::make(*graph, *pre_grad); |
| 51 | |
| 52 | if (m_weight_decay != .0f) { |
| 53 | symbol_grad = symbol_grad + m_weight_decay * symbol_weight; |
| 54 | } |
| 55 | |
| 56 | if (m_momentum != .0f) { |
| 57 | symbol_pre_grad = |
| 58 | opr::AddUpdate::make(symbol_pre_grad, symbol_grad, {m_momentum, 1.0f}); |
| 59 | return opr::AddUpdate::make(symbol_weight, -symbol_pre_grad, {1.f, m_lr}); |
| 60 | } else { |
| 61 | return opr::AddUpdate::make(symbol_weight, -symbol_grad, {1.f, m_lr}); |
| 62 | } |
| 63 | } |
| 64 | |
| 65 | Adam::Adam( |
| 66 | float lr, float weight_decay, std::pair<float, float> betas, float eps, |
nothing calls this directly
no test coverage detected