(self, config)
| 401 | |
| 402 | class BertSelfOutput(nn.Module): |
| 403 | def __init__(self, config): |
| 404 | super(BertSelfOutput, self).__init__() |
| 405 | if hasattr(config, 'deep_init') and config.deep_init: |
| 406 | init_method = scaled_init_method(mean=0.0, |
| 407 | std=config.initializer_range, |
| 408 | num_layers=config.num_hidden_layers) |
| 409 | else: |
| 410 | init_method = normal_init_method(mean=0.0, |
| 411 | std=config.initializer_range) |
| 412 | self.dense = nn.Linear(config.hidden_size, config.hidden_size, bias=True) |
| 413 | # self.dense = mpu.RowParallelLinear( |
| 414 | # input_size=config.hidden_size, |
| 415 | # output_size=config.hidden_size, |
| 416 | # bias=True, |
| 417 | # input_is_parallel=True, |
| 418 | # stride=1, |
| 419 | # init_method=init_method) |
| 420 | self.fp32_layernorm = config.fp32_layernorm |
| 421 | self.LayerNorm = BertLayerNorm(config.hidden_size, eps=config.layernorm_epsilon) |
| 422 | self.dropout = nn.Dropout(config.hidden_dropout_prob) |
| 423 | |
| 424 | def forward(self, hidden_states, input_tensor): |
| 425 | hidden_states = self.dense(hidden_states) |
nothing calls this directly
no test coverage detected