| 52 | } |
| 53 | |
| 54 | Express::VARP SGD::regularizeParameters(Express::VARP param, Express::VARP grad) { |
| 55 | VARP addWeightDecayGrad; |
| 56 | if (mRegularizationMethod == L1) { |
| 57 | auto temp = _Sign(param); |
| 58 | addWeightDecayGrad = _Const(mWeightDecay, {}, NCHW) * temp + grad; |
| 59 | } else if (mRegularizationMethod == L2) { |
| 60 | addWeightDecayGrad = _Const(mWeightDecay, {}, NCHW) * param + grad; |
| 61 | } else if (mRegularizationMethod == L1L2) { |
| 62 | auto temp = _Sign(param); |
| 63 | auto L1 = _Const(mWeightDecay, {}, NCHW) * temp; |
| 64 | auto L2 = _Const(mWeightDecay, {}, NCHW) * param; |
| 65 | addWeightDecayGrad = L1 + L2 + grad; |
| 66 | } |
| 67 | |
| 68 | return addWeightDecayGrad; |
| 69 | } |
| 70 | |
| 71 | Express::VARP SGD::onComputeUpdateValue(Express::VARP param, Express::VARP grad) { |
| 72 | auto lr = _Const(mLearningRate, {}, NCHW); |