Set weight decay coefficient, zero for bias and layernorm, 1e-1 for rest
(params)
| 57 | |
| 58 | |
| 59 | def set_weight_decay(params): |
| 60 | """ |
| 61 | Set weight decay coefficient, zero for bias and layernorm, 1e-1 for rest |
| 62 | """ |
| 63 | decay_filter = lambda x: 'layernorm' not in x.name.lower() and "bias" not in x.name.lower() |
| 64 | decay_params = list(filter(decay_filter, params)) |
| 65 | other_params = list(filter(lambda x: not decay_filter(x), params)) |
| 66 | group_params = [ |
| 67 | {"params": decay_params, "weight_decay": 1e-1}, |
| 68 | {"params": other_params, "weight_decay": 0.0}, |
| 69 | {"order_params": params}, |
| 70 | ] |
| 71 | return group_params |
| 72 | |
| 73 | |
| 74 | def add_checkpoint_callback_policy(args_param, callback, rank_id): |
no outgoing calls
no test coverage detected