| 172 | self.dropout = tf_keras.layers.Dropout(0.1) |
| 173 | |
| 174 | def call(self, inputs, training=False): |
| 175 | attention_mask = inputs.get('attention_mask') |
| 176 | mask_lengths = tf.expand_dims(tf.reduce_sum(attention_mask, axis=1), 1) |
| 177 | attention_mask = tf.tile( |
| 178 | tf.expand_dims(attention_mask, 2), [1, 1, self.num_filters] |
| 179 | ) |
| 180 | encoded = self.encoder(inputs) |
| 181 | encoded = self.norm(self.activation(self.linear_trans(encoded))) |
| 182 | encoded = self.dropout(encoded, training=training) |
| 183 | |
| 184 | mean_pooling = ( |
| 185 | tf.reduce_sum(encoded[:, 1:, :] * attention_mask, axis=1) / mask_lengths |
| 186 | ) |
| 187 | return self.linear_clas(mean_pooling) |