| 112 | |
| 113 | |
| 114 | def get_learning_rate(batch): |
| 115 | lr_wu = batch * BATCH_SIZE / WARMUP_STEP * BASE_LEARNING_RATE |
| 116 | learning_rate = tf.train.exponential_decay( |
| 117 | BASE_LEARNING_RATE / DECAY_RATE, # Base learning rate. |
| 118 | batch * BATCH_SIZE, # Current index into the dataset. |
| 119 | DECAY_STEP, # Decay step. |
| 120 | DECAY_RATE, # Decay rate. |
| 121 | staircase=True) |
| 122 | learning_rate = tf.minimum(learning_rate, lr_wu) |
| 123 | learning_rate = tf.maximum(learning_rate, 0.000001) # CLIP THE LEARNING RATE! |
| 124 | return learning_rate |
| 125 | |
| 126 | |
| 127 | def get_bn_decay(batch): |