(self, num_embeddings, embedding_dim,
init_method=init.xavier_normal_,
keep_master_weight_for_test=False)
| 138 | init_method: method to initialize weights. |
| 139 | """ |
| 140 | def __init__(self, num_embeddings, embedding_dim, |
| 141 | init_method=init.xavier_normal_, |
| 142 | keep_master_weight_for_test=False): |
| 143 | super(ParallelEmbedding, self).__init__() |
| 144 | # Keep the input dimensions. |
| 145 | self.num_embeddings = num_embeddings |
| 146 | self.embedding_dim = embedding_dim |
| 147 | # Set some detauls for compatibility. |
| 148 | self.padding_idx = None |
| 149 | self.max_norm = None |
| 150 | self.norm_type = 2. |
| 151 | self.scale_grad_by_freq = False |
| 152 | self.sparse = False |
| 153 | self._weight = None |
| 154 | # Divide the weight matrix along the embedding dimension. |
| 155 | world_size = get_model_parallel_world_size() |
| 156 | self.embedding_dim_per_partition = divide(self.embedding_dim, |
| 157 | world_size) |
| 158 | |
| 159 | # Allocate weights. |
| 160 | self.weight = Parameter(torch.Tensor(self.num_embeddings, |
| 161 | self.embedding_dim_per_partition)) |
| 162 | self.weight.model_parallel = True |
| 163 | # And initialize. |
| 164 | _initialize_affine_weight( |
| 165 | self.weight, self.num_embeddings, self.embedding_dim, |
| 166 | self.embedding_dim_per_partition, 1, init_method, |
| 167 | stride=1, return_master_weight=False) |
| 168 | |
| 169 | def forward(self, input_): |
| 170 | input_parallel = copy_to_model_parallel_region(input_) |
no test coverage detected