| 6 | |
| 7 | |
| 8 | def param_groups_weight_decay(model: nn.Module, |
| 9 | weight_decay=1e-5, |
| 10 | no_weight_decay_list=()): |
| 11 | no_weight_decay_list = set(no_weight_decay_list) |
| 12 | decay = [] |
| 13 | no_decay = [] |
| 14 | for name, param in model.named_parameters(): |
| 15 | if not param.requires_grad: |
| 16 | continue |
| 17 | |
| 18 | if param.ndim <= 1 or name.endswith( |
| 19 | '.bias') or any(nd in name for nd in no_weight_decay_list): |
| 20 | no_decay.append(param) |
| 21 | else: |
| 22 | decay.append(param) |
| 23 | |
| 24 | return [ |
| 25 | { |
| 26 | 'params': no_decay, |
| 27 | 'weight_decay': 0.0 |
| 28 | }, |
| 29 | { |
| 30 | 'params': decay, |
| 31 | 'weight_decay': weight_decay |
| 32 | }, |
| 33 | ] |
| 34 | |
| 35 | |
| 36 | def build_optimizer(optim_config, lr_scheduler_config, epochs, step_each_epoch, |