Initialize affine weight for model parallel. Build the master weight on all processes and scatter the relevant chunk.
(weight, output_size, input_size,
per_partition_size, partition_dim, init_method,
stride=1, return_master_weight=False)
| 34 | |
| 35 | |
| 36 | def _initialize_affine_weight(weight, output_size, input_size, |
| 37 | per_partition_size, partition_dim, init_method, |
| 38 | stride=1, return_master_weight=False): |
| 39 | """Initialize affine weight for model parallel. |
| 40 | |
| 41 | Build the master weight on all processes and scatter |
| 42 | the relevant chunk.""" |
| 43 | # If we only use 1 process for model parallelism, bypass scatter. |
| 44 | world_size = get_model_parallel_world_size() |
| 45 | if world_size == 1: |
| 46 | init_method(weight) |
| 47 | if return_master_weight: |
| 48 | return weight |
| 49 | return None |
| 50 | |
| 51 | # Initialize master weight |
| 52 | master_weight = torch.empty(output_size, input_size, |
| 53 | dtype=weight.dtype, |
| 54 | requires_grad=False) |
| 55 | init_method(master_weight) |
| 56 | |
| 57 | # Split and copy |
| 58 | per_partition_per_stride_size = divide(per_partition_size, stride) |
| 59 | weight_list = torch.split(master_weight, per_partition_per_stride_size, |
| 60 | dim=partition_dim) |
| 61 | rank = get_model_parallel_rank() |
| 62 | my_weight_list = weight_list[rank::world_size] |
| 63 | |
| 64 | with torch.no_grad(): |
| 65 | torch.cat(my_weight_list, dim=partition_dim, out=weight) |
| 66 | if return_master_weight: |
| 67 | return master_weight |
| 68 | return None |
| 69 | |
| 70 | |
| 71 | class VocabParallelEmbedding(torch.nn.Module): |
no test coverage detected