Initialize affine weight for model parallel. Build the master weight on all processes and scatter the relevant chunk.
(weight, output_size, input_size,
per_partition_size, partition_dim, init_method,
stride=1, return_master_weight=False, module=None, name=None, self=None)
| 36 | |
| 37 | |
| 38 | def _initialize_affine_weight(weight, output_size, input_size, |
| 39 | per_partition_size, partition_dim, init_method, |
| 40 | stride=1, return_master_weight=False, module=None, name=None, self=None): |
| 41 | """Initialize affine weight for model parallel. |
| 42 | |
| 43 | Build the master weight on all processes and scatter |
| 44 | the relevant chunk.""" |
| 45 | # If we only use 1 process for model parallelism, bypass scatter. |
| 46 | world_size = get_model_parallel_world_size() |
| 47 | if world_size == 1: |
| 48 | init_method(weight, module=module, name=name) |
| 49 | if return_master_weight: |
| 50 | return weight |
| 51 | return None |
| 52 | |
| 53 | # Initialize master weight |
| 54 | master_weight = torch.empty(output_size, input_size, |
| 55 | dtype=weight.dtype, |
| 56 | requires_grad=False, |
| 57 | device=weight.device) |
| 58 | init_method(master_weight, module=module, name=name) |
| 59 | weight_list = self.partition(full_weight=master_weight) |
| 60 | rank = get_model_parallel_rank() |
| 61 | |
| 62 | with torch.no_grad(): |
| 63 | weight.copy_(weight_list[rank]) |
| 64 | del weight_list |
| 65 | if return_master_weight: |
| 66 | return master_weight |
| 67 | return None |
| 68 | |
| 69 | |
| 70 | class VocabParallelEmbedding(torch.nn.Module): |
no test coverage detected