Function _get_data_parallel_world_size

deepspeed/utils/groups.py:684–695

Return world size for the data parallel group.

()

def _get_data_parallel_world_size():
    """Return world size for the data parallel group."""
    assert dist.is_initialized(), 'dist is not initialized'
    if mesh_device is not None:
        return dist.get_world_size(mesh_device.get_group(mesh_dim="data_parallel"))
    global mpu
    if mpu is not None:
        if hasattr(mpu, 'initialize_sequence_parallel'):
            return None
        else:
            return mpu.get_data_parallel_world_size()
    return dist.get_world_size(group=_get_data_parallel_group())


def _get_model_parallel_world_size():