| 362 | |
| 363 | |
| 364 | class Adam(OptimizerBase): |
| 365 | def __init__( |
| 366 | self, |
| 367 | lr=0.001, |
| 368 | decay1=0.9, |
| 369 | decay2=0.999, |
| 370 | eps=1e-7, |
| 371 | clip_norm=None, |
| 372 | lr_scheduler=None, |
| 373 | **kwargs |
| 374 | ): |
| 375 | """ |
| 376 | Adam (adaptive moment estimation) optimization algorithm. |
| 377 | |
| 378 | Notes |
| 379 | ----- |
| 380 | Designed to combine the advantages of :class:`AdaGrad`, which works |
| 381 | well with sparse gradients, and :class:`RMSProp`, which works well in |
| 382 | online and non-stationary settings. |
| 383 | |
| 384 | Parameters |
| 385 | ---------- |
| 386 | lr : float |
| 387 | Learning rate for update. This parameter is ignored if using |
| 388 | :class:`~numpy_ml.neural_nets.schedulers.NoamScheduler`. |
| 389 | Default is 0.001. |
| 390 | decay1 : float |
| 391 | The rate of decay to use for in running estimate of the first |
| 392 | moment (mean) of the gradient. Default is 0.9. |
| 393 | decay2 : float |
| 394 | The rate of decay to use for in running estimate of the second |
| 395 | moment (variance) of the gradient. Default is 0.999. |
| 396 | eps : float |
| 397 | Constant term to avoid divide-by-zero errors during the update |
| 398 | calc. Default is 1e-7. |
| 399 | clip_norm : float |
| 400 | If not None, all param gradients are scaled to have maximum l2 norm of |
| 401 | `clip_norm` before computing update. Default is None. |
| 402 | lr_scheduler : str, or :doc:`Scheduler <numpy_ml.neural_nets.schedulers>` object, or None |
| 403 | The learning rate scheduler. If None, use a constant learning rate |
| 404 | equal to `lr`. Default is None. |
| 405 | """ |
| 406 | super().__init__(lr, lr_scheduler) |
| 407 | |
| 408 | self.cache = {} |
| 409 | self.hyperparameters = { |
| 410 | "id": "Adam", |
| 411 | "lr": lr, |
| 412 | "eps": eps, |
| 413 | "decay1": decay1, |
| 414 | "decay2": decay2, |
| 415 | "clip_norm": clip_norm, |
| 416 | "lr_scheduler": str(self.lr_scheduler), |
| 417 | } |
| 418 | |
| 419 | def __str__(self): |
| 420 | H = self.hyperparameters |
| 421 | lr, d1, d2 = H["lr"], H["decay1"], H["decay2"] |
no outgoing calls
no test coverage detected