Function create_hcg

llm/gpt-3/utils.py:120–128

(strategy, hcg_name="HybridCommunicateGroup")

def create_hcg(strategy, hcg_name="HybridCommunicateGroup"):
    if hcg_name == "HybridCommunicateGroup":
        fleet.init(is_collective=True, strategy=strategy)
        hcg = fleet.get_hybrid_communicate_group()
    else:
        dist.init_parallel_env()
        hcg = eval("{}".format(hcg_name))(strategy)

    return hcg


def init_dist_env(