MCPcopy Create free account
hub / github.com/google-research/language / setup

Method setup

language/mentionmemory/encoders/bert_encoder.py:72–116  ·  view source on GitHub ↗
(self)

Source from the content-addressed store, hash-verified

70 layer_norm_epsilon: float = default_values.layer_norm_epsilon
71
72 def setup(self):
73
74 self.embedder = embedding.DictEmbed({
75 'token_ids':
76 embedding.Embed(
77 num_embeddings=self.vocab_size,
78 embedding_dim=self.hidden_size,
79 dtype=self.dtype,
80 embedding_init=self.kernel_init,
81 ),
82 'position_ids':
83 embedding.Embed(
84 num_embeddings=self.max_positions,
85 embedding_dim=self.hidden_size,
86 dtype=self.dtype,
87 embedding_init=self.kernel_init,
88 ),
89 'segment_ids':
90 embedding.Embed(
91 num_embeddings=self.num_segments,
92 embedding_dim=self.hidden_size,
93 dtype=self.dtype,
94 embedding_init=self.kernel_init,
95 )
96 })
97
98 self.embeddings_layer_norm = nn.LayerNorm(epsilon=self.layer_norm_epsilon)
99 self.embeddings_dropout = nn.Dropout(rate=self.dropout_rate)
100
101 self.encoder = transformer.TransformerBlock(
102 num_layers=self.num_layers,
103 model_dim=self.hidden_size,
104 intermediate_dim=self.intermediate_dim,
105 num_heads=self.num_attention_heads,
106 dropout_rate=self.dropout_rate,
107 dtype=self.dtype,
108 kernel_init=self.kernel_init,
109 bias_init=self.bias_init,
110 layer_norm_epsilon=self.layer_norm_epsilon,
111 )
112
113 self.mention_projector = nn.Dense(
114 features=self.mention_encoding_dim,
115 dtype=self.dtype,
116 )
117
118 def forward(
119 self,

Callers

nothing calls this directly

Calls

no outgoing calls

Tested by

no test coverage detected