(self)
| 38 | print(f'learning rate decaying style {self.decay_style}, ratio {self.decay_ratio}') |
| 39 | |
| 40 | def get_lr(self): |
| 41 | # https://openreview.net/pdf?id=BJYwwY9ll pg. 4 |
| 42 | if self.warmup_iter > 0 and self.num_iters <= self.warmup_iter: |
| 43 | return float(self.start_lr) * self.num_iters / self.warmup_iter |
| 44 | else: |
| 45 | if self.decay_style == self.DECAY_STYLES[0]: |
| 46 | decay_step_ratio = (self.num_iters - self.warmup_iter) / self.end_iter |
| 47 | return self.start_lr - self.start_lr * (1 - 1 / self.decay_ratio) * decay_step_ratio |
| 48 | elif self.decay_style == self.DECAY_STYLES[1]: |
| 49 | decay_step_ratio = min(1.0, (self.num_iters - self.warmup_iter) / self.end_iter) |
| 50 | return self.start_lr / self.decay_ratio * ( |
| 51 | (math.cos(math.pi * decay_step_ratio) + 1) * (self.decay_ratio - 1) / 2 + 1) |
| 52 | elif self.decay_style == self.DECAY_STYLES[2]: |
| 53 | # TODO: implement exponential decay |
| 54 | return self.start_lr |
| 55 | else: |
| 56 | return self.start_lr |
| 57 | |
| 58 | def step(self, step_num=None): |
| 59 | if step_num is None: |
no outgoing calls