Function update_learning_rate

bing_bert/deepspeed_train.py:237–249

(config, current_global_step, optimizer)

def update_learning_rate(config, current_global_step, optimizer):
    global last_global_step_from_restore

    global_step_for_lr = current_global_step - last_global_step_from_restore
    lr_this_step = config["training"]["learning_rate"] * warmup_linear_decay_exp(global_step_for_lr,
                                                                                config["training"]["decay_rate"],
                                                                                config["training"]["decay_step"],
                                                                                config["training"]["total_training_steps"],
                                                                                config["training"]["warmup_proportion"])
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr_this_step

    return lr_this_step

def report_step_metrics(args, lr, loss, step, data_sample_count):
    ##### Record the LR against global_step on tensorboard #####