| 235 | """ |
| 236 | |
| 237 | def __init__(self, |
| 238 | learning_rate, |
| 239 | end_learning_rate, |
| 240 | warmup_steps, |
| 241 | decay_steps, |
| 242 | power=1.0, |
| 243 | use_cosine=True): |
| 244 | super(LearningRate, self).__init__() |
| 245 | self.warmup_flag = False |
| 246 | if warmup_steps > 0: |
| 247 | self.warmup_flag = True |
| 248 | self.warmup_lr = WarmUpLR(learning_rate, warmup_steps) |
| 249 | self.decay_lr = PolynomialDecayLR(learning_rate, end_learning_rate, |
| 250 | decay_steps, power) |
| 251 | self.cosine_decay_lr = CosineDecayLR(end_learning_rate, learning_rate, |
| 252 | decay_steps) |
| 253 | self.warmup_steps = Tensor(np.array([warmup_steps]).astype(np.float32)) |
| 254 | |
| 255 | self.greater = P.Greater() |
| 256 | self.one = Tensor(np.array([1.0]).astype(np.float32)) |
| 257 | self.cast = P.Cast() |
| 258 | self.print = P.Print() |
| 259 | self.use_cosine = use_cosine |
| 260 | |
| 261 | def construct(self, global_step): |
| 262 | """dynamic learning rate""" |