Method no_sync

deepspeed/runtime/engine.py:2897–2914

r""" Context manager to disable gradient reduction during backward pass. This context manager has the following effects on other DeepSpeed features: 1. Incompatible with ZeRO stage 2/3 which rely on reduction for gradient partitioning. 2. It is illegal

(self)

    @contextmanager
    def no_sync(self):
        r"""
            Context manager to disable gradient reduction during backward pass.
            This context manager has the following effects on other DeepSpeed features:
            1. Incompatible with ZeRO stage 2/3 which rely on reduction for gradient partitioning.
            2. It is illegal to call engine.step() within the context manager.
            3. Tracking of gradient accumulation steps is disabled.
        """
        assert not self.zero_optimization_partition_gradients(), \
        f"no_sync context manager is incompatible with gradient partitioning logic of ZeRO stage {self.zero_optimization_stage()}"

        assert not self.inside_no_sync_ctxt, "no_sync context manager reentry is unsupported"

        self.inside_no_sync_ctxt = True
        try:
            yield
        finally:
            self.inside_no_sync_ctxt = False

    @contextmanager
    def coalesce_grad_reduction(self):