(
self,
module,
device_ids=None,
output_device=None,
dim=0,
broadcast_buffers=True,
process_group=None,
bucket_cap_mb=25,
find_unused_parameters=False,
check_reduction=False,
gradient_as_bucket_view=False,
static_graph=False,
delay_all_reduce_named_params=None,
param_to_hook_all_reduce=None,
mixed_precision: Optional[_MixedPrecision] = None,
device_mesh=None,
)
| 620 | _active_ddp_module = None |
| 621 | |
| 622 | def __init__( |
| 623 | self, |
| 624 | module, |
| 625 | device_ids=None, |
| 626 | output_device=None, |
| 627 | dim=0, |
| 628 | broadcast_buffers=True, |
| 629 | process_group=None, |
| 630 | bucket_cap_mb=25, |
| 631 | find_unused_parameters=False, |
| 632 | check_reduction=False, |
| 633 | gradient_as_bucket_view=False, |
| 634 | static_graph=False, |
| 635 | delay_all_reduce_named_params=None, |
| 636 | param_to_hook_all_reduce=None, |
| 637 | mixed_precision: Optional[_MixedPrecision] = None, |
| 638 | device_mesh=None, |
| 639 | ): |
| 640 | super().__init__() |
| 641 | Joinable.__init__(self) |
| 642 | self.logger = None |
| 643 | if bool(delay_all_reduce_named_params is not None) != bool( |
| 644 | param_to_hook_all_reduce is not None |
| 645 | ): |
| 646 | self._log_and_throw( |
| 647 | ValueError, |
| 648 | "delay_all_reduce_named_params and param_to_hook_all_reduce " |
| 649 | "need to be set at the same time.", |
| 650 | ) |
| 651 | |
| 652 | self._delay_all_reduce_params = [] |
| 653 | if hasattr(module, "_ddp_params_and_buffers_to_ignore"): |
| 654 | self.parameters_to_ignore = set(module._ddp_params_and_buffers_to_ignore) |
| 655 | else: |
| 656 | self.parameters_to_ignore = set() |
| 657 | if delay_all_reduce_named_params is not None: |
| 658 | for name, param in delay_all_reduce_named_params: |
| 659 | self.parameters_to_ignore.add(name) |
| 660 | self._delay_all_reduce_params.append(param) |
| 661 | |
| 662 | self._module_parameters = [ |
| 663 | p |
| 664 | for n, p in module.named_parameters() |
| 665 | if n not in self.parameters_to_ignore |
| 666 | ] |
| 667 | if not any(p.requires_grad for p in self._module_parameters): |
| 668 | if len(self._delay_all_reduce_params): |
| 669 | logger.info("Delay the AllReduce of all parameters.") |
| 670 | else: |
| 671 | self._log_and_throw( |
| 672 | RuntimeError, |
| 673 | "DistributedDataParallel is not needed when a module " |
| 674 | "doesn't have any parameter that requires a gradient.", |
| 675 | ) |
| 676 | |
| 677 | if device_ids is not None and len(device_ids) > 1: |
| 678 | self._log_and_throw( |
| 679 | ValueError, |
nothing calls this directly
no test coverage detected