| 24 | |
| 25 | |
| 26 | def group_weight(weight_group, module, norm_layer, lr): |
| 27 | group_decay = [] |
| 28 | group_no_decay = [] |
| 29 | count = 0 |
| 30 | for m in module.modules(): |
| 31 | if isinstance(m, nn.Linear): |
| 32 | group_decay.append(m.weight) |
| 33 | if m.bias is not None: |
| 34 | group_no_decay.append(m.bias) |
| 35 | elif isinstance(m, (nn.Conv1d, nn.Conv2d, nn.Conv3d, nn.ConvTranspose2d, nn.ConvTranspose3d)): |
| 36 | group_decay.append(m.weight) |
| 37 | if m.bias is not None: |
| 38 | group_no_decay.append(m.bias) |
| 39 | elif ( |
| 40 | isinstance(m, norm_layer) |
| 41 | or isinstance(m, nn.BatchNorm1d) |
| 42 | or isinstance(m, nn.BatchNorm2d) |
| 43 | or isinstance(m, nn.BatchNorm3d) |
| 44 | or isinstance(m, nn.GroupNorm) |
| 45 | or isinstance(m, nn.LayerNorm) |
| 46 | # or isinstance(m, LayerNorm) |
| 47 | ): |
| 48 | if m.weight is not None: |
| 49 | group_no_decay.append(m.weight) |
| 50 | if m.bias is not None: |
| 51 | group_no_decay.append(m.bias) |
| 52 | elif isinstance(m, nn.Parameter): |
| 53 | group_decay.append(m) |
| 54 | |
| 55 | # assert len(list(module.parameters())) >= len(group_decay) + len(group_no_decay) |
| 56 | print( |
| 57 | "Weight Decay:", |
| 58 | len(group_decay), |
| 59 | "Weight No Decay:", |
| 60 | len(group_no_decay), |
| 61 | "Total:", |
| 62 | len(list(module.parameters())), |
| 63 | ) |
| 64 | # for i in list(module.parameters()): |
| 65 | # print(type(i), i.size()) |
| 66 | # assert 1==2 |
| 67 | weight_group.append(dict(params=group_decay, lr=lr)) |
| 68 | weight_group.append(dict(params=group_no_decay, weight_decay=0.0, lr=lr)) |
| 69 | return weight_group |
| 70 | |
| 71 | |
| 72 | def configure_optimizers(model, lr, weight_decay): |