(params, coalesce=True, bucket_size_mb=-1)
| 29 | |
| 30 | |
| 31 | def allreduce_grads(params, coalesce=True, bucket_size_mb=-1): |
| 32 | grads = [ |
| 33 | param.grad.data for param in params |
| 34 | if param.requires_grad and param.grad is not None |
| 35 | ] |
| 36 | world_size = dist.get_world_size() |
| 37 | if coalesce: |
| 38 | _allreduce_coalesced(grads, world_size, bucket_size_mb) |
| 39 | else: |
| 40 | for tensor in grads: |
| 41 | dist.all_reduce(tensor.div_(world_size)) |
| 42 | |
| 43 | |
| 44 | class DistOptimizerHook(OptimizerHook): |
nothing calls this directly
no test coverage detected