| 164 | |
| 165 | |
| 166 | def build_ds_config(config, args): |
| 167 | opt_lower = config.TRAIN.OPTIMIZER.NAME.lower() |
| 168 | if opt_lower == 'adamw': |
| 169 | optimizer = { |
| 170 | 'type': 'AdamW', |
| 171 | 'params': { |
| 172 | 'lr': config.TRAIN.BASE_LR, |
| 173 | 'eps': config.TRAIN.OPTIMIZER.EPS, |
| 174 | 'betas': config.TRAIN.OPTIMIZER.BETAS, |
| 175 | 'weight_decay': config.TRAIN.WEIGHT_DECAY |
| 176 | } |
| 177 | } |
| 178 | else: |
| 179 | return NotImplemented |
| 180 | |
| 181 | ds_config = { |
| 182 | 'train_micro_batch_size_per_gpu': config.DATA.BATCH_SIZE, |
| 183 | 'optimizer': optimizer, |
| 184 | 'bf16': { |
| 185 | 'enabled': True, |
| 186 | }, |
| 187 | 'zero_optimization': { |
| 188 | 'stage': 1, |
| 189 | 'allgather_partitions': True, |
| 190 | 'allgather_bucket_size': 1e9, |
| 191 | 'overlap_comm': True, |
| 192 | 'reduce_scatter': True, |
| 193 | 'reduce_bucket_size': 1e9, |
| 194 | 'contiguous_gradients': True |
| 195 | }, |
| 196 | 'steps_per_print': 1e10, |
| 197 | 'gradient_accumulation_steps': config.TRAIN.ACCUMULATION_STEPS, |
| 198 | 'gradient_clipping': config.TRAIN.CLIP_GRAD, |
| 199 | } |
| 200 | return ds_config |
| 201 | |
| 202 | |
| 203 | @torch.no_grad() |