Get default param list for optimizer Args: overrides (dict: str -> (dict: str -> float)): if not `None`, provides values for optimizer hyperparameters (LR, weight decay) for module parameters with a given name; e.g. {"embedding": {"lr": 0.01, "we
(
model: torch.nn.Module,
base_lr,
weight_decay,
weight_decay_norm,
bias_lr_factor=1.0,
weight_decay_bias=None,
overrides: Optional[Dict[str, Dict[str, float]]] = None,
)
| 125 | |
| 126 | |
| 127 | def get_default_optimizer_params( |
| 128 | model: torch.nn.Module, |
| 129 | base_lr, |
| 130 | weight_decay, |
| 131 | weight_decay_norm, |
| 132 | bias_lr_factor=1.0, |
| 133 | weight_decay_bias=None, |
| 134 | overrides: Optional[Dict[str, Dict[str, float]]] = None, |
| 135 | ): |
| 136 | """ |
| 137 | Get default param list for optimizer |
| 138 | |
| 139 | Args: |
| 140 | overrides (dict: str -> (dict: str -> float)): |
| 141 | if not `None`, provides values for optimizer hyperparameters |
| 142 | (LR, weight decay) for module parameters with a given name; e.g. |
| 143 | {"embedding": {"lr": 0.01, "weight_decay": 0.1}} will set the LR and |
| 144 | weight decay values for all module parameters named `embedding` (default: None) |
| 145 | """ |
| 146 | if weight_decay_bias is None: |
| 147 | weight_decay_bias = weight_decay |
| 148 | norm_module_types = ( |
| 149 | torch.nn.BatchNorm1d, |
| 150 | torch.nn.BatchNorm2d, |
| 151 | torch.nn.BatchNorm3d, |
| 152 | torch.nn.SyncBatchNorm, |
| 153 | # NaiveSyncBatchNorm inherits from BatchNorm2d |
| 154 | torch.nn.GroupNorm, |
| 155 | torch.nn.InstanceNorm1d, |
| 156 | torch.nn.InstanceNorm2d, |
| 157 | torch.nn.InstanceNorm3d, |
| 158 | torch.nn.LayerNorm, |
| 159 | torch.nn.LocalResponseNorm, |
| 160 | ) |
| 161 | params: List[Dict[str, Any]] = [] |
| 162 | memo: Set[torch.nn.parameter.Parameter] = set() |
| 163 | for module in model.modules(): |
| 164 | for module_param_name, value in module.named_parameters(recurse=False): |
| 165 | if not value.requires_grad: |
| 166 | continue |
| 167 | # Avoid duplicating parameters |
| 168 | if value in memo: |
| 169 | continue |
| 170 | memo.add(value) |
| 171 | |
| 172 | schedule_params = { |
| 173 | "lr": base_lr, |
| 174 | "weight_decay": weight_decay, |
| 175 | } |
| 176 | if isinstance(module, norm_module_types): |
| 177 | schedule_params["weight_decay"] = weight_decay_norm |
| 178 | elif module_param_name == "bias": |
| 179 | # NOTE: unlike Detectron v1, we now default BIAS_LR_FACTOR to 1.0 |
| 180 | # and WEIGHT_DECAY_BIAS to WEIGHT_DECAY so that bias optimizer |
| 181 | # hyperparameters are by default exactly the same as for regular |
| 182 | # weights. |
| 183 | schedule_params["lr"] = base_lr * bias_lr_factor |
| 184 | schedule_params["weight_decay"] = weight_decay_bias |
no test coverage detected