Initialize variables needed for distributed training.
(cfg)
| 268 | |
| 269 | |
| 270 | def init_distributed_training(cfg): |
| 271 | """ |
| 272 | Initialize variables needed for distributed training. |
| 273 | """ |
| 274 | if cfg.NUM_GPUS <= 1: |
| 275 | return |
| 276 | num_gpus_per_machine = cfg.NUM_GPUS |
| 277 | num_machines = dist.get_world_size() // num_gpus_per_machine |
| 278 | for i in range(num_machines): |
| 279 | ranks_on_i = list( |
| 280 | range(i * num_gpus_per_machine, (i + 1) * num_gpus_per_machine) |
| 281 | ) |
| 282 | pg = dist.new_group(ranks_on_i) |
| 283 | if i == cfg.SHARD_ID: |
| 284 | global _LOCAL_PROCESS_GROUP |
| 285 | _LOCAL_PROCESS_GROUP = pg |
| 286 | |
| 287 | |
| 288 | def get_local_size() -> int: |
nothing calls this directly
no outgoing calls
no test coverage detected