Calculate the global rank corresponding to the first local rank in the tensor model parallel group.
()
| 343 | |
| 344 | |
| 345 | def get_tensor_model_parallel_src_rank(): |
| 346 | """Calculate the global rank corresponding to the first local rank |
| 347 | in the tensor model parallel group.""" |
| 348 | global_rank = torch.distributed.get_rank() |
| 349 | local_world_size = get_tensor_model_parallel_world_size() |
| 350 | return (global_rank // local_world_size) * local_world_size |
| 351 | |
| 352 | |
| 353 | def get_pipeline_model_parallel_first_rank(): |
no test coverage detected