Calculate the global rank corresponding to a local rank zeor in the model parallel group.
()
| 110 | |
| 111 | |
| 112 | def get_model_parallel_src_rank(): |
| 113 | """Calculate the global rank corresponding to a local rank zeor |
| 114 | in the model parallel group.""" |
| 115 | global_rank = torch.distributed.get_rank() |
| 116 | local_world_size = get_model_parallel_world_size() |
| 117 | return (global_rank // local_world_size) * local_world_size |
| 118 | |
| 119 | |
| 120 | def get_data_parallel_world_size(): |
no test coverage detected