| 105 | |
| 106 | |
| 107 | def scale_learning_rate(config, num_processes): |
| 108 | # linear scale the learning rate according to total batch size, may not be optimal |
| 109 | linear_scaled_lr = config.TRAIN.BASE_LR * config.DATA.BATCH_SIZE * num_processes / 512.0 |
| 110 | linear_scaled_warmup_lr = config.TRAIN.WARMUP_LR * config.DATA.BATCH_SIZE * num_processes / 512.0 |
| 111 | linear_scaled_min_lr = config.TRAIN.MIN_LR * config.DATA.BATCH_SIZE * num_processes / 512.0 |
| 112 | # gradient accumulation also need to scale the learning rate |
| 113 | if config.TRAIN.ACCUMULATION_STEPS > 1: |
| 114 | linear_scaled_lr = linear_scaled_lr * config.TRAIN.ACCUMULATION_STEPS |
| 115 | linear_scaled_warmup_lr = linear_scaled_warmup_lr * config.TRAIN.ACCUMULATION_STEPS |
| 116 | linear_scaled_min_lr = linear_scaled_min_lr * config.TRAIN.ACCUMULATION_STEPS |
| 117 | config.defrost() |
| 118 | config.TRAIN.BASE_LR = linear_scaled_lr |
| 119 | config.TRAIN.WARMUP_LR = linear_scaled_warmup_lr |
| 120 | config.TRAIN.MIN_LR = linear_scaled_min_lr |
| 121 | config.freeze() |
| 122 | |
| 123 | logger.info('BASE_LR={}'.format(config.TRAIN.BASE_LR)) |
| 124 | logger.info('WARMUP_LR={}'.format(config.TRAIN.WARMUP_LR)) |
| 125 | logger.info('MIN_LR={}'.format(config.TRAIN.MIN_LR)) |
| 126 | |
| 127 | |
| 128 | def log_model_statistic(model_wo_ddp): |