(self, config: Qwen2Config)
| 125 | supports_gradient_checkpointing = True |
| 126 | |
| 127 | def __init__(self, config: Qwen2Config): |
| 128 | super().__init__(config) |
| 129 | self.layers = nn.ModuleList( |
| 130 | [Qwen2EncoderLayer(config, i) for i in range(self.config.num_hidden_layers)] |
| 131 | ) |
| 132 | if config.rope: |
| 133 | self.rotary_emb = Qwen2RotaryEmbedding(config=config) |
| 134 | else: |
| 135 | self.rotary_emb = None |
| 136 | if hasattr(config, "norm") and config.norm: |
| 137 | self.norm = Qwen2RMSNorm(config.hidden_size, eps=config.rms_norm_eps) |
| 138 | else: |
| 139 | self.norm = None |
| 140 | self.gradient_checkpointing = True |
| 141 | self.post_init() |
| 142 | |
| 143 | def _init_weights(self, module): |
| 144 | std = self.config.initializer_range |
no test coverage detected