| 400 | |
| 401 | |
| 402 | class BertSelfOutput(nn.Module): |
| 403 | def __init__(self, config): |
| 404 | super(BertSelfOutput, self).__init__() |
| 405 | if hasattr(config, 'deep_init') and config.deep_init: |
| 406 | init_method = scaled_init_method(mean=0.0, |
| 407 | std=config.initializer_range, |
| 408 | num_layers=config.num_hidden_layers) |
| 409 | else: |
| 410 | init_method = normal_init_method(mean=0.0, |
| 411 | std=config.initializer_range) |
| 412 | self.dense = nn.Linear(config.hidden_size, config.hidden_size, bias=True) |
| 413 | # self.dense = mpu.RowParallelLinear( |
| 414 | # input_size=config.hidden_size, |
| 415 | # output_size=config.hidden_size, |
| 416 | # bias=True, |
| 417 | # input_is_parallel=True, |
| 418 | # stride=1, |
| 419 | # init_method=init_method) |
| 420 | self.fp32_layernorm = config.fp32_layernorm |
| 421 | self.LayerNorm = BertLayerNorm(config.hidden_size, eps=config.layernorm_epsilon) |
| 422 | self.dropout = nn.Dropout(config.hidden_dropout_prob) |
| 423 | |
| 424 | def forward(self, hidden_states, input_tensor): |
| 425 | hidden_states = self.dense(hidden_states) |
| 426 | hidden_states = self.dropout(hidden_states) |
| 427 | ln_input = hidden_states + input_tensor |
| 428 | previous_type = ln_input.type() |
| 429 | if self.fp32_layernorm: |
| 430 | ln_input = ln_input.float() |
| 431 | hidden_states = self.LayerNorm(ln_input) |
| 432 | if self.fp32_layernorm: |
| 433 | hidden_states = hidden_states.type(previous_type) |
| 434 | return hidden_states |
| 435 | |
| 436 | |
| 437 | class BertAttention(nn.Module): |