(cls, cfg, model)
| 586 | |
| 587 | |
| 588 | def build_optimizer(cls, cfg, model): |
| 589 | cfg_solver = cfg['SOLVER'] |
| 590 | weight_decay_norm = cfg_solver['WEIGHT_DECAY_NORM'] |
| 591 | weight_decay_embed = cfg_solver['WEIGHT_DECAY_EMBED'] |
| 592 | weight_decay_bias = cfg_solver.get('WEIGHT_DECAY_BIAS', 0.0) |
| 593 | |
| 594 | defaults = {} |
| 595 | defaults["lr"] = cfg_solver['BASE_LR'] |
| 596 | defaults["weight_decay"] = cfg_solver['WEIGHT_DECAY'] |
| 597 | |
| 598 | norm_module_types = ( |
| 599 | torch.nn.BatchNorm1d, |
| 600 | torch.nn.BatchNorm2d, |
| 601 | torch.nn.BatchNorm3d, |
| 602 | torch.nn.SyncBatchNorm, |
| 603 | # NaiveSyncBatchNorm inherits from BatchNorm2d |
| 604 | torch.nn.GroupNorm, |
| 605 | torch.nn.InstanceNorm1d, |
| 606 | torch.nn.InstanceNorm2d, |
| 607 | torch.nn.InstanceNorm3d, |
| 608 | torch.nn.LayerNorm, |
| 609 | torch.nn.LocalResponseNorm, |
| 610 | ) |
| 611 | |
| 612 | lr_multiplier = cfg['SOLVER']['LR_MULTIPLIER'] |
| 613 | params: List[Dict[str, Any]] = [] |
| 614 | memo: Set[torch.nn.parameter.Parameter] = set() |
| 615 | for module_name, module in model.named_modules(): |
| 616 | for module_param_name, value in module.named_parameters(recurse=False): |
| 617 | if not value.requires_grad: |
| 618 | continue |
| 619 | # Avoid duplicating parameters |
| 620 | if value in memo: |
| 621 | continue |
| 622 | memo.add(value) |
| 623 | |
| 624 | hyperparams = copy.copy(defaults) |
| 625 | |
| 626 | for key, lr_mul in lr_multiplier.items(): |
| 627 | if key in "{}.{}".format(module_name, module_param_name): |
| 628 | hyperparams["lr"] = hyperparams["lr"] * lr_mul |
| 629 | if is_main_process(): |
| 630 | logger.info("Modify Learning rate of {}: {}".format( |
| 631 | "{}.{}".format(module_name, module_param_name), lr_mul)) |
| 632 | |
| 633 | if ( |
| 634 | "relative_position_bias_table" in module_param_name |
| 635 | or "absolute_pos_embed" in module_param_name |
| 636 | ): |
| 637 | hyperparams["weight_decay"] = 0.0 |
| 638 | if isinstance(module, norm_module_types): |
| 639 | hyperparams["weight_decay"] = weight_decay_norm |
| 640 | if isinstance(module, torch.nn.Embedding): |
| 641 | hyperparams["weight_decay"] = weight_decay_embed |
| 642 | if "bias" in module_name: |
| 643 | hyperparams["weight_decay"] = weight_decay_bias |
| 644 | params.append({"params": [value], **hyperparams}) |
| 645 |
nothing calls this directly
no test coverage detected