r"""Apply loss scaler for manual backward pass. Use this method when calling loss.backward() directly instead of engine.backward(). This applies the appropriate loss scaler for mixed precision training, allowing you to manually control the backward pass while still benefitin
(self, loss)
| 3096 | optimizer.independent_gradient_partition_epilogue() |
| 3097 | |
| 3098 | def scale(self, loss): |
| 3099 | r"""Apply loss scaler for manual backward pass. |
| 3100 | |
| 3101 | Use this method when calling loss.backward() directly instead of engine.backward(). |
| 3102 | This applies the appropriate loss scaler for mixed precision training, allowing you |
| 3103 | to manually control the backward pass while still benefiting from DeepSpeed's |
| 3104 | gradient scaling functionality. |
| 3105 | |
| 3106 | Example:: |
| 3107 | |
| 3108 | output = engine(input) |
| 3109 | loss = criterion(output, target) |
| 3110 | scaled_loss = engine.scale(loss) |
| 3111 | scaled_loss.backward() # Manual backward call |
| 3112 | engine.step() |
| 3113 | |
| 3114 | Arguments: |
| 3115 | loss: Scalar loss tensor to be scaled |
| 3116 | |
| 3117 | Returns: |
| 3118 | Scaled loss tensor ready for .backward() call |
| 3119 | |
| 3120 | Raises: |
| 3121 | RuntimeError: If AMP (NVIDIA Apex) is enabled. AMP requires using engine.backward() |
| 3122 | directly as it uses a context manager that cannot be separated from |
| 3123 | the backward call. |
| 3124 | AssertionError: If loss is not a scalar tensor with grad_fn, or if no optimizer |
| 3125 | is configured. |
| 3126 | """ |
| 3127 | assert self.optimizer is not None and not isinstance(self.optimizer, DummyOptim), \ |
| 3128 | "must provide optimizer during init in order to use scale" |
| 3129 | assert maybe_loss_for_backward(loss), \ |
| 3130 | "loss must be a scalar tensor with grad_fn. For non-scalar tensors, use tensor.backward(grad)" |
| 3131 | |
| 3132 | # AMP (NVIDIA Apex) uses a context manager that wraps both scaling and backward, |
| 3133 | # so it cannot be used with manual backward calls |
| 3134 | if self.amp_enabled(): |
| 3135 | raise RuntimeError("engine.scale() is not compatible with AMP (NVIDIA Apex). " |
| 3136 | "When using AMP, you must call engine.backward(loss) instead of manual backward.") |
| 3137 | |
| 3138 | # Apply loss scaler based on optimizer type |
| 3139 | scaled_loss = loss |
| 3140 | if isinstance(self.optimizer, ZeROOptimizer): |
| 3141 | scaled_loss = self.optimizer.scale_if_loss(scaled_loss) |
| 3142 | elif self.torch_autocast_z0_gradscaler: |
| 3143 | scaled_loss = self.torch_autocast_z0_gradscaler.scale(scaled_loss) |
| 3144 | |
| 3145 | # Mark that scale() was called for validation in backward hook |
| 3146 | self._manual_backward_expected = True |
| 3147 | |
| 3148 | return scaled_loss |
| 3149 | |
| 3150 | @instrument_w_nvtx |
| 3151 | def backward(self, loss, retain_graph=False, scale_wrt_gas=True): |
no test coverage detected