(self)
| 131 | assert_allclose(test_outputs_segment * mask, ref_outputs_segment * mask) |
| 132 | |
| 133 | def test_dropout_rate(self): |
| 134 | hidden_dim = 12 |
| 135 | num_heads = 4 |
| 136 | vocab_size = 24 |
| 137 | source_length = 11 |
| 138 | dropout_rate = 0.1 |
| 139 | num_layers = 2 |
| 140 | encoder = Encoder.default_config().set( |
| 141 | dim=hidden_dim, |
| 142 | vocab_size=vocab_size, |
| 143 | dropout_rate=dropout_rate, |
| 144 | emb=bert_embedding_config(type_vocab_size=1, max_position_embeddings=source_length), |
| 145 | transformer=bert_transformer_config(num_layers=num_layers, num_heads=num_heads), |
| 146 | pad_token_id=0, |
| 147 | ) |
| 148 | encoder.param_init = DefaultInitializer.default_config().set( |
| 149 | init_by_param_name={ |
| 150 | PARAM_REGEXP_WEIGHT: WeightInitializer.default_config().set( |
| 151 | fan=None, scale=0.02, distribution="normal" |
| 152 | ) |
| 153 | } |
| 154 | ) |
| 155 | set_layer_norm_eps_recursively(encoder, 1e-5) |
| 156 | layer_test = encoder.set(name="layer_test").instantiate(parent=None) |
| 157 | self.assertEqual(layer_test.emb.dropout.config.rate, dropout_rate) |
| 158 | for i in range(num_layers): |
| 159 | transformer_layer = getattr(layer_test.transformer, f"layer{i}") |
| 160 | self.assertEqual(transformer_layer.self_attention.dropout.config.rate, dropout_rate) |
| 161 | self.assertEqual(transformer_layer.feed_forward.dropout.config.rate, dropout_rate) |
| 162 | |
| 163 | |
| 164 | class TestCausalEncoder(TestCase): |
nothing calls this directly
no test coverage detected