Function warmup_linear_decay_linear

pretrain_src/optimization_utils.py:67–75

(global_step,
                          total_steps,
                          warmup=0.002,
                          )

    return (1.0 - x)**degree

def warmup_linear_decay_linear(global_step,
                          total_steps,
                          warmup=0.002,
                          ):
    x = global_step / total_steps # x is the propotion of the updated steps wrt total steps
    if x < warmup:
        return x / warmup
    else:
        return 1.0 - (x - warmup)/(1.0 - warmup)