Paged Lion optimizer. Arguments: params (`torch.tensor`): The input parameters to optimize. lr (`float`, defaults to 1e-4): The learning rate. betas (`tuple(float, float)`, defaults to (0.9, 0.999)):
(
self,
params,
lr=1e-4,
betas=(0.9, 0.99),
weight_decay=0,
optim_bits=32,
args=None,
min_8bit_size=4096,
)
| 142 | |
| 143 | class PagedLion(Optimizer1State): |
| 144 | def __init__( |
| 145 | self, |
| 146 | params, |
| 147 | lr=1e-4, |
| 148 | betas=(0.9, 0.99), |
| 149 | weight_decay=0, |
| 150 | optim_bits=32, |
| 151 | args=None, |
| 152 | min_8bit_size=4096, |
| 153 | ): |
| 154 | """ |
| 155 | Paged Lion optimizer. |
| 156 | |
| 157 | Arguments: |
| 158 | params (`torch.tensor`): |
| 159 | The input parameters to optimize. |
| 160 | lr (`float`, defaults to 1e-4): |
| 161 | The learning rate. |
| 162 | betas (`tuple(float, float)`, defaults to (0.9, 0.999)): |
| 163 | The beta values are the decay rates of the first and second-order moment of the optimizer. |
| 164 | weight_decay (`float`, defaults to 0): |
| 165 | The weight decay value for the optimizer. |
| 166 | optim_bits (`int`, defaults to 32): |
| 167 | The number of bits of the optimizer state. |
| 168 | args (`object`, defaults to `None`): |
| 169 | An object with additional arguments. |
| 170 | min_8bit_size (`int`, defaults to 4096): |
| 171 | The minimum number of elements of the parameter tensors for 8-bit optimization. |
| 172 | """ |
| 173 | super().__init__( |
| 174 | "lion", |
| 175 | params, |
| 176 | lr, |
| 177 | betas, |
| 178 | 0.0, |
| 179 | weight_decay, |
| 180 | optim_bits, |
| 181 | args, |
| 182 | min_8bit_size, |
| 183 | is_paged=True, |
| 184 | ) |
| 185 | |
| 186 | |
| 187 | class PagedLion8bit(Optimizer1State): |