(cls)
| 279 | |
| 280 | @classmethod |
| 281 | def default_config(cls) -> Config: |
| 282 | cfg = super().default_config() |
| 283 | # TODO(xianzhi): verify the model initialization method. |
| 284 | cfg.param_init = param_init.DefaultInitializer.default_config().set( |
| 285 | init_by_param_name={ |
| 286 | PARAM_REGEXP_WEIGHT: param_init.WeightInitializer.default_config().set( |
| 287 | fan="fan_avg", distribution="uniform" |
| 288 | ) |
| 289 | } |
| 290 | ) |
| 291 | # pylint: disable=no-member |
| 292 | cfg.visual_pos_emb.param_init = GaussianInitializer.default_config().set(std=0.02) |
| 293 | transformer_layer_cfg = cfg.transformer.layer |
| 294 | transformer_layer_cfg.feed_forward.activation = "nn.gelu" |
| 295 | transformer_layer_cfg.feed_forward.norm = layer_norm_config() |
| 296 | transformer_layer_cfg.feed_forward.hidden_dim = scaled_hidden_dim(4) |
| 297 | transformer_layer_cfg.self_attention.norm = layer_norm_config() |
| 298 | # pylint: enable=no-member |
| 299 | return cfg |
| 300 | |
| 301 | def _create_layer_parameter_specs(self) -> dict[str, ParameterSpec]: |
| 302 | cfg = self.config |
no test coverage detected