(self)
| 996 | f'Client Optimizer (type = {type(self.client_optimizer)} is not instantiated but Client LR Scheduler is instantiated' |
| 997 | |
| 998 | def _broadcast_model(self): |
| 999 | |
| 1000 | def is_replicated(p): |
| 1001 | if hasattr(p, "ds_status") and p.ds_status is not ZeroParamStatus.AVAILABLE: |
| 1002 | return False |
| 1003 | return True |
| 1004 | |
| 1005 | for p in self.module.parameters(): |
| 1006 | # Broadcast the model for different parameters |
| 1007 | if is_moe_param(p): |
| 1008 | if torch.is_tensor(p) and is_replicated(p): |
| 1009 | dist.broadcast(p, |
| 1010 | groups._get_expert_broadcast_src_rank(p.group_name), |
| 1011 | group=self.expert_data_parallel_group[p.group_name]) |
| 1012 | else: |
| 1013 | if torch.is_tensor(p) and is_replicated(p): |
| 1014 | dist.broadcast(p, groups._get_broadcast_src_rank(), group=self.data_parallel_group) |
| 1015 | |
| 1016 | @staticmethod |
| 1017 | def __check_params(model: Module, dtype: torch.dtype) -> None: |
no test coverage detected