Paged 32-bit Lion optimizer. Arguments: params (`torch.tensor`): The input parameters to optimize. lr (`float`, defaults to 1e-4): The learning rate. betas (`tuple(float, float)`, defaults to (0.9, 0.999)):
(
self,
params,
lr=1e-4,
betas=(0.9, 0.99),
weight_decay=0,
args=None,
min_8bit_size=4096,
)
| 227 | |
| 228 | class PagedLion32bit(Optimizer1State): |
| 229 | def __init__( |
| 230 | self, |
| 231 | params, |
| 232 | lr=1e-4, |
| 233 | betas=(0.9, 0.99), |
| 234 | weight_decay=0, |
| 235 | args=None, |
| 236 | min_8bit_size=4096, |
| 237 | ): |
| 238 | """ |
| 239 | Paged 32-bit Lion optimizer. |
| 240 | |
| 241 | Arguments: |
| 242 | params (`torch.tensor`): |
| 243 | The input parameters to optimize. |
| 244 | lr (`float`, defaults to 1e-4): |
| 245 | The learning rate. |
| 246 | betas (`tuple(float, float)`, defaults to (0.9, 0.999)): |
| 247 | The beta values are the decay rates of the first and second-order moment of the optimizer. |
| 248 | weight_decay (`float`, defaults to 0): |
| 249 | The weight decay value for the optimizer. |
| 250 | args (`object`, defaults to `None`): |
| 251 | An object with additional arguments. |
| 252 | min_8bit_size (`int`, defaults to 4096): |
| 253 | The minimum number of elements of the parameter tensors for 8-bit optimization. |
| 254 | """ |
| 255 | super().__init__( |
| 256 | "lion", |
| 257 | params, |
| 258 | lr, |
| 259 | betas, |
| 260 | 0.0, |
| 261 | weight_decay, |
| 262 | 32, |
| 263 | args, |
| 264 | min_8bit_size, |
| 265 | is_paged=True, |
| 266 | ) |