(self,
num_layers,
hidden_size,
num_attention_heads,
max_sequence_length,
max_memory_length,
embedding_dropout_prob,
attention_dropout_prob,
output_dropout_prob,
checkpoint_activations,
checkpoint_num_layers=1,
layernorm_epsilon=1.0e-5,
init_method_std=0.02,
use_scaled_init_for_output_weights=True,
relative_encoding=False,
block_position_encoding=False,
performer=False,
use_decoder_layer=False,
attention_scale=1.0,
)
| 636 | """ |
| 637 | |
| 638 | def __init__(self, |
| 639 | num_layers, |
| 640 | hidden_size, |
| 641 | num_attention_heads, |
| 642 | max_sequence_length, |
| 643 | max_memory_length, |
| 644 | embedding_dropout_prob, |
| 645 | attention_dropout_prob, |
| 646 | output_dropout_prob, |
| 647 | checkpoint_activations, |
| 648 | checkpoint_num_layers=1, |
| 649 | layernorm_epsilon=1.0e-5, |
| 650 | init_method_std=0.02, |
| 651 | use_scaled_init_for_output_weights=True, |
| 652 | relative_encoding=False, |
| 653 | block_position_encoding=False, |
| 654 | performer=False, |
| 655 | use_decoder_layer=False, |
| 656 | attention_scale=1.0, |
| 657 | ): |
| 658 | super(GPT2ParallelTransformer, self).__init__() |
| 659 | self.hidden_size = hidden_size |
| 660 | # Store activation checkpoiting flag. |
| 661 | self.checkpoint_activations = checkpoint_activations |
| 662 | self.checkpoint_num_layers = checkpoint_num_layers |
| 663 | self.max_memory_length = max_memory_length |
| 664 | self.performer = performer |
| 665 | self.use_decoder_layer = use_decoder_layer |
| 666 | assert not (performer and relative_encoding) |
| 667 | |
| 668 | output_layer_init_method = None |
| 669 | if use_scaled_init_for_output_weights: |
| 670 | output_layer_init_method = scaled_init_method(init_method_std, |
| 671 | num_layers) |
| 672 | # Embeddings dropout |
| 673 | self.embedding_dropout = torch.nn.Dropout(embedding_dropout_prob) |
| 674 | self.relative_encoding = relative_encoding |
| 675 | self.block_position_encoding = block_position_encoding |
| 676 | if relative_encoding: |
| 677 | # Relative position embedding |
| 678 | self.position_embeddings = PositionalEmbedding(hidden_size) |
| 679 | # Per attention head and per partition values. |
| 680 | world_size = get_model_parallel_world_size() |
| 681 | self.hidden_size_per_attention_head = divide(hidden_size, |
| 682 | num_attention_heads) |
| 683 | self.num_attention_heads_per_partition = divide(num_attention_heads, |
| 684 | world_size) |
| 685 | self.r_w_bias = torch.nn.Parameter( |
| 686 | torch.Tensor(self.num_attention_heads_per_partition, self.hidden_size_per_attention_head)) |
| 687 | self.r_w_bias.model_parallel = True |
| 688 | self.r_r_bias = torch.nn.Parameter( |
| 689 | torch.Tensor(self.num_attention_heads_per_partition, self.hidden_size_per_attention_head)) |
| 690 | self.r_r_bias.model_parallel = True |
| 691 | # Always initialize bias to zero. |
| 692 | with torch.no_grad(): |
| 693 | self.r_w_bias.zero_() |
| 694 | self.r_r_bias.zero_() |
| 695 | else: |
nothing calls this directly
no test coverage detected