Forward step for passed-in model. If first stage, input tensor is obtained from data_iterator, otherwise passed-in input_tensor is used. Returns output tensor.
(forward_step_func, data_iterator, model, input_tensor, losses_reduced)
| 41 | |
| 42 | |
| 43 | def forward_step(forward_step_func, data_iterator, model, input_tensor, losses_reduced): |
| 44 | """Forward step for passed-in model. |
| 45 | |
| 46 | If first stage, input tensor is obtained from data_iterator, otherwise |
| 47 | passed-in input_tensor is used. |
| 48 | |
| 49 | Returns output tensor.""" |
| 50 | timers = get_timers() |
| 51 | |
| 52 | args = get_args() |
| 53 | |
| 54 | timers("forward-compute").start() |
| 55 | unwrapped_model = unwrap_model(model, (torchDDP, LocalDDP, Float16Module)) |
| 56 | if not args.deepspeed: |
| 57 | unwrapped_model.set_input_tensor(input_tensor) |
| 58 | else: |
| 59 | unwrapped_model.module.set_input_tensor(input_tensor) |
| 60 | |
| 61 | output_tensor, loss_func = forward_step_func(data_iterator, model) |
| 62 | if mpu.is_pipeline_last_stage(): |
| 63 | output_tensor = loss_func(output_tensor) |
| 64 | loss, loss_reduced = output_tensor |
| 65 | output_tensor = loss / get_num_microbatches() |
| 66 | losses_reduced.append(loss_reduced) |
| 67 | timers("forward-compute").stop() |
| 68 | |
| 69 | return output_tensor |
| 70 | |
| 71 | |
| 72 | def backward_step( |
no test coverage detected