r"""Execute backward pass on the loss Arguments: loss: Torch tensor on which to execute backward propagation allreduce_gradients: is deprecated, ignored, and will soon be removed' retain_graph: bool, default: false forward on user defined c
(self, loss, allreduce_gradients=True, release_loss=False, retain_graph=False, scale_wrt_gas=True)
| 1820 | |
| 1821 | @instrument_w_nvtx |
| 1822 | def backward(self, loss, allreduce_gradients=True, release_loss=False, retain_graph=False, scale_wrt_gas=True): |
| 1823 | r"""Execute backward pass on the loss |
| 1824 | Arguments: |
| 1825 | loss: Torch tensor on which to execute backward propagation |
| 1826 | allreduce_gradients: is deprecated, ignored, and will soon be removed' |
| 1827 | retain_graph: bool, default: false |
| 1828 | forward on user defined choice of retain_graph |
| 1829 | """ |
| 1830 | |
| 1831 | see_memory_usage("Engine before backward", force=self.memory_breakdown()) |
| 1832 | |
| 1833 | if self.scale_wrt_gas is not None: |
| 1834 | scale_wrt_gas = self.scale_wrt_gas |
| 1835 | |
| 1836 | if not allreduce_gradients: |
| 1837 | logger.warning(f"Argument `allreduce_gradients` is deprecated, ignored, and will soon be removed") |
| 1838 | |
| 1839 | # scale loss w.r.t. gradient accumulation if needed |
| 1840 | if self.gradient_accumulation_steps() > 1 and scale_wrt_gas: |
| 1841 | loss = self._scale_loss_by_gas(loss.float()) |
| 1842 | |
| 1843 | # Log training Loss |
| 1844 | if self.monitor.enabled: |
| 1845 | if self.is_gradient_accumulation_boundary(): |
| 1846 | if self.global_rank == 0: |
| 1847 | self.summary_events = [( |
| 1848 | f"Train/Samples/train_loss", |
| 1849 | sum(self.losses) / self.gradient_accumulation_steps(), |
| 1850 | self.global_samples, |
| 1851 | )] |
| 1852 | self.monitor.write_events(self.summary_events) |
| 1853 | |
| 1854 | if self.is_gradient_accumulation_boundary(): |
| 1855 | self.losses = [] |
| 1856 | else: |
| 1857 | self.losses.append(loss.mean().item()) |
| 1858 | |
| 1859 | self._start_timers(self.engine_timers.backward_timers) |
| 1860 | |
| 1861 | assert self.optimizer is not None and not isinstance(self.optimizer, DummyOptim), \ |
| 1862 | "must provide optimizer during init in order to use backward" |
| 1863 | |
| 1864 | self._start_timers(self.engine_timers.backward_inner_timers) |
| 1865 | |
| 1866 | if self.zero_optimization(): |
| 1867 | self.optimizer.is_gradient_accumulation_boundary = self.is_gradient_accumulation_boundary() |
| 1868 | self.optimizer.backward(loss, retain_graph=retain_graph) |
| 1869 | elif self.amp_enabled(): |
| 1870 | # AMP requires delaying unscale when inside gradient accumulation boundaries |
| 1871 | # https://nvidia.github.io/apex/advanced.html#gradient-accumulation-across-iterations |
| 1872 | delay_unscale = not self.is_gradient_accumulation_boundary() |
| 1873 | with amp.scale_loss(loss, self.optimizer, delay_unscale=delay_unscale) as scaled_loss: |
| 1874 | scaled_loss.backward(retain_graph=retain_graph) |
| 1875 | elif self.fp16_enabled(): |
| 1876 | if self.eigenvalue_enabled(): |
| 1877 | self.optimizer.backward(loss, create_graph=True, retain_graph=True) |
| 1878 | else: |
| 1879 | self.optimizer.backward(loss, retain_graph=retain_graph) |
nothing calls this directly
no test coverage detected