optimization funxtion
(loss,
warmup_steps,
num_train_steps,
learning_rate,
train_program,
weight_decay,
scheduler='linear_warmup_decay',
use_fp16=False,
use_dynamic_loss_scaling=False,
init_loss_scaling=1.0,
beta1=0.9,
beta2=0.98,
epsilon=1e-06,
boundaries=None,
values=None)
| 50 | |
| 51 | |
| 52 | def optimization(loss, |
| 53 | warmup_steps, |
| 54 | num_train_steps, |
| 55 | learning_rate, |
| 56 | train_program, |
| 57 | weight_decay, |
| 58 | scheduler='linear_warmup_decay', |
| 59 | use_fp16=False, |
| 60 | use_dynamic_loss_scaling=False, |
| 61 | init_loss_scaling=1.0, |
| 62 | beta1=0.9, |
| 63 | beta2=0.98, |
| 64 | epsilon=1e-06, |
| 65 | boundaries=None, |
| 66 | values=None): |
| 67 | """optimization funxtion""" |
| 68 | def exclude_from_weight_decay(name): |
| 69 | """exclude from weight decay""" |
| 70 | name = name.rstrip('.master') |
| 71 | if name.find("layer_norm") > -1: |
| 72 | return True |
| 73 | bias_suffix = ["_bias", "_b", ".b_0"] |
| 74 | for suffix in bias_suffix: |
| 75 | if name.endswith(suffix): |
| 76 | return True |
| 77 | return False |
| 78 | |
| 79 | if warmup_steps > 0: |
| 80 | if scheduler == 'noam_decay': |
| 81 | scheduled_lr = fluid.layers.learning_rate_scheduler \ |
| 82 | .noam_decay(1 / (warmup_steps * (learning_rate ** 2)), |
| 83 | warmup_steps) |
| 84 | elif scheduler == 'linear_warmup_decay': |
| 85 | scheduled_lr = linear_warmup_decay(learning_rate, warmup_steps, |
| 86 | num_train_steps) |
| 87 | elif scheduler == 'scale_by_epoch_decay': |
| 88 | if boundaries is None: |
| 89 | boundaries = [10000, 20000] |
| 90 | if values is None: |
| 91 | values = [5e-6, 5e-7, 5e-8] |
| 92 | scheduled_lr = fluid.layers.piecewise_decay(boundaries=boundaries, values=values) |
| 93 | else: |
| 94 | raise ValueError("Unkown learning rate scheduler, should be " |
| 95 | "'noam_decay' or 'linear_warmup_decay'") |
| 96 | optimizer = fluid.optimizer.Adam(learning_rate=scheduled_lr, beta1=beta1, beta2=beta2, epsilon=epsilon) |
| 97 | else: |
| 98 | scheduled_lr = fluid.layers.create_global_var( |
| 99 | name=fluid.unique_name.generate("learning_rate"), |
| 100 | shape=[1], |
| 101 | value=learning_rate, |
| 102 | dtype='float32', |
| 103 | persistable=True) |
| 104 | optimizer = fluid.optimizer.Adam(learning_rate=scheduled_lr, beta1=beta1, beta2=beta2, epsilon=epsilon) |
| 105 | optimizer._learning_rate_map[fluid.default_main_program()] = scheduled_lr |
| 106 | |
| 107 | if use_fp16: |
| 108 | optimizer = fluid.contrib.mixed_precision.decorator.decorate(optimizer, |
| 109 | amp_lists=fluid.contrib.mixed_precision.AutoMixedPrecisionLists( |
no test coverage detected