(self)
| 70 | layer_norm_epsilon: float = default_values.layer_norm_epsilon |
| 71 | |
| 72 | def setup(self): |
| 73 | |
| 74 | self.embedder = embedding.DictEmbed({ |
| 75 | 'token_ids': |
| 76 | embedding.Embed( |
| 77 | num_embeddings=self.vocab_size, |
| 78 | embedding_dim=self.hidden_size, |
| 79 | dtype=self.dtype, |
| 80 | embedding_init=self.kernel_init, |
| 81 | ), |
| 82 | 'position_ids': |
| 83 | embedding.Embed( |
| 84 | num_embeddings=self.max_positions, |
| 85 | embedding_dim=self.hidden_size, |
| 86 | dtype=self.dtype, |
| 87 | embedding_init=self.kernel_init, |
| 88 | ), |
| 89 | 'segment_ids': |
| 90 | embedding.Embed( |
| 91 | num_embeddings=self.num_segments, |
| 92 | embedding_dim=self.hidden_size, |
| 93 | dtype=self.dtype, |
| 94 | embedding_init=self.kernel_init, |
| 95 | ) |
| 96 | }) |
| 97 | |
| 98 | self.embeddings_layer_norm = nn.LayerNorm(epsilon=self.layer_norm_epsilon) |
| 99 | self.embeddings_dropout = nn.Dropout(rate=self.dropout_rate) |
| 100 | |
| 101 | self.encoder = transformer.TransformerBlock( |
| 102 | num_layers=self.num_layers, |
| 103 | model_dim=self.hidden_size, |
| 104 | intermediate_dim=self.intermediate_dim, |
| 105 | num_heads=self.num_attention_heads, |
| 106 | dropout_rate=self.dropout_rate, |
| 107 | dtype=self.dtype, |
| 108 | kernel_init=self.kernel_init, |
| 109 | bias_init=self.bias_init, |
| 110 | layer_norm_epsilon=self.layer_norm_epsilon, |
| 111 | ) |
| 112 | |
| 113 | self.mention_projector = nn.Dense( |
| 114 | features=self.mention_encoding_dim, |
| 115 | dtype=self.dtype, |
| 116 | ) |
| 117 | |
| 118 | def forward( |
| 119 | self, |
nothing calls this directly
no outgoing calls
no test coverage detected