(self)
| 683 | logger.info(f"use_sequence_parallel_moe: {self.use_sequence_parallel_moe}") |
| 684 | |
| 685 | def set_communicate_group(self): |
| 686 | # different tp group id |
| 687 | # prevent different tp_groups using the same group_id |
| 688 | tp_gid_offset = envs.FD_TP_GROUP_GID_OFFSET |
| 689 | dist.collective._set_custom_gid(self.data_parallel_rank + tp_gid_offset) |
| 690 | |
| 691 | self.tp_group = dist.new_group( |
| 692 | range( |
| 693 | self.data_parallel_rank * self.tensor_parallel_size, |
| 694 | (self.data_parallel_rank + 1) * self.tensor_parallel_size, |
| 695 | ) |
| 696 | ) |
| 697 | dist.collective._set_custom_gid(None) |
| 698 | # same ep group id |
| 699 | if self.enable_expert_parallel: |
| 700 | dist.collective._set_custom_gid(self.data_parallel_size + tp_gid_offset) |
| 701 | self.ep_group = dist.new_group(range(self.expert_parallel_size)) |
| 702 | dist.collective._set_custom_gid(None) |
| 703 | logger.info( |
| 704 | f"data_parallel_size: {self.data_parallel_size}, tensor_parallel_size: {self.tensor_parallel_size}, expert_parallel_size: {self.expert_parallel_size}, data_parallel_rank: {self.data_parallel_rank}, tensor_parallel_rank: {self.tensor_parallel_rank}, expert_parallel_rank: {self.expert_parallel_rank}, tp_group: {self.tp_group}." |
| 705 | ) |
| 706 | |
| 707 | def print(self): |
| 708 | """ |
no test coverage detected