(self)
| 1591 | f'Client Optimizer (type = {type(self.client_optimizer)} is not instantiated but Client LR Scheduler is instantiated' |
| 1592 | |
| 1593 | def _broadcast_model(self): |
| 1594 | if self.dist_backend is None: |
| 1595 | return |
| 1596 | |
| 1597 | def is_replicated(p): |
| 1598 | if hasattr(p, "ds_status") and p.ds_status is not ZeroParamStatus.AVAILABLE: |
| 1599 | return False |
| 1600 | elif hasattr(p, 'ds_optim_param'): |
| 1601 | # do not broadcast OptimizedLinear parameters, they are unique per base weight shard |
| 1602 | return False |
| 1603 | return True |
| 1604 | |
| 1605 | for n, p in self.module.named_parameters(): |
| 1606 | # Broadcast the model for different parameters |
| 1607 | if is_moe_param(p): |
| 1608 | if torch.is_tensor(p) and is_replicated(p): |
| 1609 | dist.broadcast(p.data, |
| 1610 | groups._get_expert_broadcast_src_rank(p.group_name), |
| 1611 | group=self.expert_data_parallel_group[p.group_name]) |
| 1612 | else: |
| 1613 | if torch.is_tensor(p) and is_replicated(p): |
| 1614 | dist.broadcast(p.data, groups._get_broadcast_src_rank(), group=self.seq_data_parallel_group) |
| 1615 | |
| 1616 | @staticmethod |
| 1617 | def __check_params(model: Module, dtype: torch.dtype) -> None: |
no test coverage detected