Method
__init__
(
self,
world_size=1,
rank=0,
gpus_per_node=8,
*,
cp_size=1,
cp_config=None,
tp_size=1,
pp_size=1,
pp_partition=None,
moe_cluster_size=-1, # -1 means no moe
moe_tp_size=-1, # -1 means no moe
moe_ep_size=-1, # -1 means no moe
attn_tp_size=-1,
attn_cp_size=-1,
enable_attention_dp=False,
enable_lm_head_tp_in_adp=False)
Source from the content-addressed store, hash-verified
| 495 | |
| 496 | # Intentionally repeated for type hints |
| 497 | def __init__( |
| 498 | self, |
| 499 | world_size=1, |
| 500 | rank=0, |
| 501 | gpus_per_node=8, |
| 502 | *, |
| 503 | cp_size=1, |
| 504 | cp_config=None, |
| 505 | tp_size=1, |
| 506 | pp_size=1, |
| 507 | pp_partition=None, |
| 508 | moe_cluster_size=-1, # -1 means no moe |
| 509 | moe_tp_size=-1, # -1 means no moe |
| 510 | moe_ep_size=-1, # -1 means no moe |
| 511 | attn_tp_size=-1, |
| 512 | attn_cp_size=-1, |
| 513 | enable_attention_dp=False, |
| 514 | enable_lm_head_tp_in_adp=False): |
| 515 | super().__init__(world_size=world_size, |
| 516 | rank=rank, |
| 517 | gpus_per_node=gpus_per_node, |
| 518 | cp_size=cp_size, |
| 519 | cp_config=cp_config, |
| 520 | tp_size=tp_size, |
| 521 | pp_size=pp_size, |
| 522 | pp_partition=pp_partition, |
| 523 | moe_cluster_size=moe_cluster_size, |
| 524 | moe_tp_size=moe_tp_size, |
| 525 | moe_ep_size=moe_ep_size, |
| 526 | attn_tp_size=attn_tp_size, |
| 527 | attn_cp_size=attn_cp_size, |
| 528 | enable_attention_dp=enable_attention_dp, |
| 529 | enable_lm_head_tp_in_adp=enable_lm_head_tp_in_adp) |
| 530 | |
| 531 | def repurpose_helix_cp_to_tp(self): |
| 532 | # In helix parallelism, CP is relevant only for the attention layer. These ranks are repurposed to TP |
Tested by
no test coverage detected