(self, layer_id: int, model_args: ModelArgs)
| 293 | """ |
| 294 | |
| 295 | def __init__(self, layer_id: int, model_args: ModelArgs): |
| 296 | super().__init__() |
| 297 | self.n_heads = model_args.n_heads |
| 298 | self.dim = model_args.dim |
| 299 | self.attention = Attention(model_args) |
| 300 | self.feed_forward = FeedForward( |
| 301 | dim=model_args.dim, |
| 302 | hidden_dim=4 * model_args.dim, |
| 303 | multiple_of=model_args.multiple_of, |
| 304 | ffn_dim_multiplier=model_args.ffn_dim_multiplier, |
| 305 | ) |
| 306 | self.layer_id = layer_id |
| 307 | self.num_layers = model_args.n_layers |
| 308 | |
| 309 | self.attention_norm = RMSNorm( |
| 310 | dim=model_args.dim, eps=model_args.norm_eps |
| 311 | ) |
| 312 | self.ffn_norm = RMSNorm( |
| 313 | dim=model_args.dim, eps=model_args.norm_eps |
| 314 | ) |
| 315 | |
| 316 | if model_args.depth_init: |
| 317 | self.weight_init_std = 0.02 / (2 * (self.layer_id + 1)) ** 0.5 |
| 318 | else: |
| 319 | self.weight_init_std = 0.02 / (2 * self.num_layers) ** 0.5 |
| 320 | |
| 321 | def forward( |
| 322 | self, |
nothing calls this directly
no test coverage detected