| 98 | return decayed_lr |
| 99 | |
| 100 | def sgd_optimizer(self, lr_policy, loss): |
| 101 | decayed_lr = self.get_lr(lr_policy) |
| 102 | optimizer = fluid.optimizer.Momentum( |
| 103 | learning_rate=decayed_lr, |
| 104 | momentum=self.momentum, |
| 105 | regularization=fluid.regularizer.L2Decay( |
| 106 | regularization_coeff=self.weight_decay), |
| 107 | ) |
| 108 | if cfg.MODEL.FP16: |
| 109 | if cfg.MODEL.MODEL_NAME in ["pspnet"]: |
| 110 | custom_black_list = {"pool2d"} |
| 111 | else: |
| 112 | custom_black_list = {} |
| 113 | amp_lists = AutoMixedPrecisionLists( |
| 114 | custom_black_list=custom_black_list) |
| 115 | assert isinstance(cfg.MODEL.SCALE_LOSS, float) or isinstance(cfg.MODEL.SCALE_LOSS, str), \ |
| 116 | "data type of MODEL.SCALE_LOSS must be float or str" |
| 117 | if isinstance(cfg.MODEL.SCALE_LOSS, float): |
| 118 | optimizer = decorate( |
| 119 | optimizer, |
| 120 | amp_lists=amp_lists, |
| 121 | init_loss_scaling=cfg.MODEL.SCALE_LOSS, |
| 122 | use_dynamic_loss_scaling=False) |
| 123 | else: |
| 124 | assert cfg.MODEL.SCALE_LOSS.lower() in [ |
| 125 | 'dynamic' |
| 126 | ], "if MODEL.SCALE_LOSS is a string,\ |
| 127 | must be set as 'DYNAMIC'!" |
| 128 | |
| 129 | optimizer = decorate( |
| 130 | optimizer, |
| 131 | amp_lists=amp_lists, |
| 132 | use_dynamic_loss_scaling=True) |
| 133 | |
| 134 | optimizer.minimize(loss) |
| 135 | return decayed_lr |
| 136 | |
| 137 | def adam_optimizer(self, lr_policy, loss): |
| 138 | decayed_lr = self.get_lr(lr_policy) |