Communicate tensors between stages. Used as helper method in other communication methods that are used in megatron/schedules.py. Takes the following arguments: tensor_send_next: tensor to send to next rank (no tensor sent if set to None). tensor_sen
(
tensor_send_next, tensor_send_prev, recv_prev, recv_next, use_ring_exchange=False
)
| 22 | |
| 23 | |
| 24 | def _communicate( |
| 25 | tensor_send_next, tensor_send_prev, recv_prev, recv_next, use_ring_exchange=False |
| 26 | ): |
| 27 | """Communicate tensors between stages. Used as helper method in other |
| 28 | communication methods that are used in megatron/schedules.py. |
| 29 | |
| 30 | Takes the following arguments: |
| 31 | tensor_send_next: tensor to send to next rank (no tensor sent if |
| 32 | set to None). |
| 33 | tensor_send_prev: tensor to send to prev rank (no tensor sent if |
| 34 | set to None). |
| 35 | recv_prev: boolean for whether tensor should be received from |
| 36 | previous rank. |
| 37 | recv_next: boolean for whether tensor should be received from |
| 38 | next rank. |
| 39 | use_ring_exchange: boolean for whether torch.distributed.ring_exchange() |
| 40 | API should be used. |
| 41 | |
| 42 | Returns: |
| 43 | (tensor_recv_prev, tensor_recv_next) |
| 44 | """ |
| 45 | args = get_args() |
| 46 | |
| 47 | # Create placeholder tensors for receive in forward and backward directions |
| 48 | # if needed. |
| 49 | tensor_recv_prev = None |
| 50 | tensor_recv_next = None |
| 51 | tensor_shape = (args.seq_length, args.micro_batch_size, args.hidden_size) |
| 52 | if args.scatter_gather_tensors_in_pipeline: |
| 53 | tensor_chunk_shape = ( |
| 54 | reduce(operator.mul, tensor_shape, 1) |
| 55 | // mpu.get_tensor_model_parallel_world_size() |
| 56 | ) |
| 57 | else: |
| 58 | tensor_chunk_shape = tensor_shape |
| 59 | dtype = args.params_dtype |
| 60 | if args.fp32_residual_connection: |
| 61 | dtype = torch.float |
| 62 | if recv_prev: |
| 63 | tensor_recv_prev = torch.empty( |
| 64 | tensor_chunk_shape, |
| 65 | requires_grad=True, |
| 66 | device=torch.cuda.current_device(), |
| 67 | dtype=dtype, |
| 68 | ) |
| 69 | if recv_next: |
| 70 | tensor_recv_next = torch.empty( |
| 71 | tensor_chunk_shape, |
| 72 | requires_grad=True, |
| 73 | device=torch.cuda.current_device(), |
| 74 | dtype=dtype, |
| 75 | ) |
| 76 | |
| 77 | # Split tensor into smaller chunks if using scatter-gather optimization. |
| 78 | if args.scatter_gather_tensors_in_pipeline: |
| 79 | if tensor_send_next is not None: |
| 80 | tensor_send_next = mpu.split_tensor_into_1d_equal_chunks(tensor_send_next) |
| 81 |
no test coverage detected