Helper function to synchronize (barrier) among all processes when using distributed training
()
| 23 | |
| 24 | |
| 25 | def synchronize(): |
| 26 | """ |
| 27 | Helper function to synchronize (barrier) among all processes when |
| 28 | using distributed training |
| 29 | """ |
| 30 | if not dist.is_available(): |
| 31 | return |
| 32 | if not dist.is_initialized(): |
| 33 | return |
| 34 | world_size = dist.get_world_size() |
| 35 | if world_size == 1: |
| 36 | return |
| 37 | dist.barrier() |
| 38 | |
| 39 | |
| 40 | @torch.no_grad() |