Optimization parameters for Adam. Args: learning_rate: a floating point value. The learning rate. beta1: A float value. The exponential decay rate for the 1st moment estimates. beta2: A float value. The exponential decay rate for the 2nd moment estimates.
(self,
learning_rate,
beta1=0.9,
beta2=0.999,
epsilon=1e-08,
lazy_adam=True,
sum_inside_sqrt=True,
use_gradient_accumulation=True,
clip_weight_min=None,
clip_weight_max=None)
| 337 | """ |
| 338 | |
| 339 | def __init__(self, |
| 340 | learning_rate, |
| 341 | beta1=0.9, |
| 342 | beta2=0.999, |
| 343 | epsilon=1e-08, |
| 344 | lazy_adam=True, |
| 345 | sum_inside_sqrt=True, |
| 346 | use_gradient_accumulation=True, |
| 347 | clip_weight_min=None, |
| 348 | clip_weight_max=None): |
| 349 | """Optimization parameters for Adam. |
| 350 | |
| 351 | Args: |
| 352 | learning_rate: a floating point value. The learning rate. |
| 353 | beta1: A float value. |
| 354 | The exponential decay rate for the 1st moment estimates. |
| 355 | beta2: A float value. |
| 356 | The exponential decay rate for the 2nd moment estimates. |
| 357 | epsilon: A small constant for numerical stability. |
| 358 | lazy_adam: Use lazy Adam instead of Adam. Lazy Adam trains faster. |
| 359 | Please see `optimization_parameters.proto` for details. |
| 360 | sum_inside_sqrt: This improves training speed. Please see |
| 361 | `optimization_parameters.proto` for details. |
| 362 | use_gradient_accumulation: setting this to `False` makes embedding |
| 363 | gradients calculation less accurate but faster. Please see |
| 364 | `optimization_parameters.proto` for details. |
| 365 | for details. |
| 366 | clip_weight_min: the minimum value to clip by; None means -infinity. |
| 367 | clip_weight_max: the maximum value to clip by; None means +infinity. |
| 368 | """ |
| 369 | super(AdamParameters, |
| 370 | self).__init__(learning_rate, use_gradient_accumulation, |
| 371 | clip_weight_min, clip_weight_max) |
| 372 | if beta1 < 0. or beta1 >= 1.: |
| 373 | raise ValueError('beta1 must be between 0. and 1; got {}.'.format(beta1)) |
| 374 | if beta2 < 0. or beta2 >= 1.: |
| 375 | raise ValueError('beta2 must be between 0. and 1; got {}.'.format(beta2)) |
| 376 | if epsilon <= 0.: |
| 377 | raise ValueError('epsilon must be positive; got {}.'.format(epsilon)) |
| 378 | if not use_gradient_accumulation and not lazy_adam: |
| 379 | raise ValueError( |
| 380 | 'When disabling Lazy Adam, gradient accumulation must be used.') |
| 381 | |
| 382 | self.beta1 = beta1 |
| 383 | self.beta2 = beta2 |
| 384 | self.epsilon = epsilon |
| 385 | self.lazy_adam = lazy_adam |
| 386 | self.sum_inside_sqrt = sum_inside_sqrt |
| 387 | |
| 388 | |
| 389 | @tf_export(v1=['tpu.experimental.StochasticGradientDescentParameters']) |