Function reduce_tensor

utils.py:91–95

(tensor)

def reduce_tensor(tensor):
    rt = tensor.clone()
    dist.all_reduce(rt, op=dist.ReduceOp.SUM)
    rt /= dist.get_world_size()
    return rt

def init_dist_slurm():
    """Initialize slurm distributed training environment.