| 108 | |
| 109 | |
| 110 | def scale_learning_rate(config, num_processes): |
| 111 | # linear scale the learning rate according to total batch size, may not be optimal |
| 112 | linear_scaled_lr = config.TRAIN.BASE_LR * \ |
| 113 | config.DATA.BATCH_SIZE * num_processes / 512.0 |
| 114 | linear_scaled_warmup_lr = config.TRAIN.WARMUP_LR * \ |
| 115 | config.DATA.BATCH_SIZE * num_processes / 512.0 |
| 116 | linear_scaled_min_lr = config.TRAIN.MIN_LR * \ |
| 117 | config.DATA.BATCH_SIZE * num_processes / 512.0 |
| 118 | # gradient accumulation also need to scale the learning rate |
| 119 | if config.TRAIN.ACCUMULATION_STEPS > 1: |
| 120 | linear_scaled_lr = linear_scaled_lr * config.TRAIN.ACCUMULATION_STEPS |
| 121 | linear_scaled_warmup_lr = linear_scaled_warmup_lr * config.TRAIN.ACCUMULATION_STEPS |
| 122 | linear_scaled_min_lr = linear_scaled_min_lr * config.TRAIN.ACCUMULATION_STEPS |
| 123 | config.defrost() |
| 124 | config.TRAIN.BASE_LR = linear_scaled_lr |
| 125 | config.TRAIN.WARMUP_LR = linear_scaled_warmup_lr |
| 126 | config.TRAIN.MIN_LR = linear_scaled_min_lr |
| 127 | config.freeze() |
| 128 | |
| 129 | logger.info('BASE_LR={}'.format(config.TRAIN.BASE_LR)) |
| 130 | logger.info('WARMUP_LR={}'.format(config.TRAIN.WARMUP_LR)) |
| 131 | logger.info('MIN_LR={}'.format(config.TRAIN.MIN_LR)) |
| 132 | |
| 133 | |
| 134 | def setup_autoresume(config): |