(params, coalesce=True, bucket_size_mb=-1)
| 32 | |
| 33 | |
| 34 | def allreduce_grads(params, coalesce=True, bucket_size_mb=-1): |
| 35 | grads = [ |
| 36 | param.grad.data for param in params |
| 37 | if param.requires_grad and param.grad is not None |
| 38 | ] |
| 39 | world_size = dist.get_world_size() |
| 40 | if coalesce: |
| 41 | _allreduce_coalesced(grads, world_size, bucket_size_mb) |
| 42 | else: |
| 43 | for tensor in grads: |
| 44 | dist.all_reduce(tensor.div_(world_size)) |
| 45 | |
| 46 | |
| 47 | class DistOptimizerHook(OptimizerHook): |
nothing calls this directly
no test coverage detected