r"""Execute forward propagation Arguments: *inputs: Variable length input list **kwargs: variable length keyword arguments
(self, *inputs, **kwargs)
| 2746 | |
| 2747 | @instrument_w_nvtx |
| 2748 | def forward(self, *inputs, **kwargs): |
| 2749 | r"""Execute forward propagation |
| 2750 | Arguments: |
| 2751 | *inputs: Variable length input list |
| 2752 | **kwargs: variable length keyword arguments |
| 2753 | """ |
| 2754 | # Clear the backward seen flag at the start of each forward pass. |
| 2755 | # This is used to track multiple gradient hook phases with reentrant checkpointing. |
| 2756 | if isinstance(self.optimizer, ZeROOptimizer): |
| 2757 | self.optimizer.clear_backward_seen_flag() |
| 2758 | |
| 2759 | if self.autotuning_profile_model_info(): |
| 2760 | ma = get_ma_status() |
| 2761 | |
| 2762 | if self.is_deepcompile_enabled() and not self.is_deepcompile_active() and not self.is_compiled: |
| 2763 | log_dist_once( |
| 2764 | "DeepCompile is enabled but engine.compile() has not been called; executing without DeepCompile until compile() runs.", |
| 2765 | ranks=[0]) |
| 2766 | |
| 2767 | if self.is_deepcompile_active() and hasattr(self, "launch_compile_passes"): |
| 2768 | # We can't have this in forward prologue as the compiler compiles hooks including the forward prologue. |
| 2769 | self.launch_compile_passes(self.global_steps) |
| 2770 | |
| 2771 | with deepcompile_z3_forward_context(self) as z3_eager_fallback, autocast_if_enabled(self): |
| 2772 | loss = self.module(*inputs, **kwargs) |
| 2773 | |
| 2774 | forward_graph_id = None |
| 2775 | |
| 2776 | def backward_prologue(): |
| 2777 | self._backward_prologue() |
| 2778 | if z3_eager_fallback is not None and forward_graph_id is not None: |
| 2779 | z3_eager_fallback.record_backward_start(forward_graph_id) |
| 2780 | |
| 2781 | # Register output backward hooks |
| 2782 | # preprocess_once_fn is called for preprocessing |
| 2783 | # preprocess_per_tensor_fn scales a tensor for gradient accumulation |
| 2784 | hook_manager = register_output_backward_hooks(loss, |
| 2785 | preprocess_once_fn=backward_prologue, |
| 2786 | preprocess_per_tensor_fn=self._backward_prologue_per_tensor) |
| 2787 | if z3_eager_fallback is not None and hook_manager.hook_handles: |
| 2788 | forward_graph_id = z3_eager_fallback.record_forward_graph() |
| 2789 | |
| 2790 | if self.autotuning_profile_model_info(): |
| 2791 | activation_mem = get_ma_status() - ma |
| 2792 | self.autotuning_model_info["activation_mem_per_gpu"] = activation_mem |
| 2793 | print_json_dist(self.autotuning_model_info, [0], path=self.autotuning_model_info_path()) |
| 2794 | exit() |
| 2795 | |
| 2796 | return loss |
| 2797 | |
| 2798 | def _cast_inputs_half(self, inputs): |
| 2799 | if isinstance(inputs, (list, tuple)): |
nothing calls this directly
no test coverage detected