| 81 | |
| 82 | |
| 83 | class IterExponential: |
| 84 | def __init__(self, total_iter_length, final_ratio, warmup_steps=0) -> None: |
| 85 | """ |
| 86 | Customized iteration-wise exponential scheduler. |
| 87 | Re-calculate for every step, to reduce error accumulation |
| 88 | |
| 89 | Args: |
| 90 | total_iter_length (int): Expected total iteration number |
| 91 | final_ratio (float): Expected LR ratio at n_iter = total_iter_length |
| 92 | """ |
| 93 | self.total_length = total_iter_length |
| 94 | self.effective_length = total_iter_length - warmup_steps |
| 95 | self.final_ratio = final_ratio |
| 96 | self.warmup_steps = warmup_steps |
| 97 | |
| 98 | def __call__(self, n_iter) -> float: |
| 99 | if n_iter < self.warmup_steps: |
| 100 | alpha = 1.0 * n_iter / self.warmup_steps |
| 101 | elif n_iter >= self.total_length: |
| 102 | alpha = self.final_ratio |
| 103 | else: |
| 104 | actual_iter = n_iter - self.warmup_steps |
| 105 | alpha = np.exp( |
| 106 | actual_iter / self.effective_length * np.log(self.final_ratio) |
| 107 | ) |
| 108 | return alpha |
| 109 | |
| 110 | |
| 111 | def get_iter_exponential_schedule(optimizer: Optimizer, num_warmup_steps: int, num_training_steps: int, final_ratio: float): |
no outgoing calls
no test coverage detected