(module)
| 211 | |
| 212 | |
| 213 | def glm_get_params_for_weight_decay_optimization(module): |
| 214 | weight_decay_params = {'params': []} |
| 215 | no_weight_decay_params = {'params': [], 'weight_decay': 0.0} |
| 216 | for module_ in module.modules(): |
| 217 | if isinstance(module_, (mpu.LayerNorm, torch.nn.LayerNorm)): |
| 218 | no_weight_decay_params['params'].extend( |
| 219 | [p for p in list(module_._parameters.values()) |
| 220 | if p is not None and p.requires_grad]) |
| 221 | else: |
| 222 | weight_decay_params['params'].extend( |
| 223 | [p for n, p in list(module_._parameters.items()) |
| 224 | if p is not None and p.requires_grad and n != 'bias']) |
| 225 | no_weight_decay_params['params'].extend( |
| 226 | [p for n, p in list(module_._parameters.items()) |
| 227 | if p is not None and p.requires_grad and n == 'bias']) |
| 228 | |
| 229 | return weight_decay_params, no_weight_decay_params |
no test coverage detected