Adds a decay to weights in the model. This is a form of L2 regularization. Args: weight_decay: strength of the regularization
(model, weight_decay)
| 56 | |
| 57 | |
| 58 | def add_weight_decay(model, weight_decay): |
| 59 | """Adds a decay to weights in the model. |
| 60 | |
| 61 | This is a form of L2 regularization. |
| 62 | |
| 63 | Args: |
| 64 | weight_decay: strength of the regularization |
| 65 | """ |
| 66 | if weight_decay <= 0.0: |
| 67 | return |
| 68 | wd = model.param_init_net.ConstantFill( |
| 69 | [], 'wd', shape=[1], value=weight_decay |
| 70 | ) |
| 71 | ONE = model.param_init_net.ConstantFill([], "ONE", shape=[1], value=1.0) |
| 72 | for param in _get_weights(model): |
| 73 | # Equivalent to: grad += wd * param |
| 74 | grad = model.param_to_grad[param] |
| 75 | model.net.WeightedSum( |
| 76 | [grad, ONE, param, wd], |
| 77 | grad, |
| 78 | ) |
nothing calls this directly
no test coverage detected
searching dependent graphs…