| 227 | class ChatGLMConfig(PretrainedConfig): |
| 228 | model_type = "chatglm" |
| 229 | def __init__( |
| 230 | self, |
| 231 | num_layers=28, |
| 232 | padded_vocab_size=65024, |
| 233 | hidden_size=4096, |
| 234 | ffn_hidden_size=13696, |
| 235 | kv_channels=128, |
| 236 | num_attention_heads=32, |
| 237 | seq_length=2048, |
| 238 | hidden_dropout=0.0, |
| 239 | classifier_dropout=None, |
| 240 | attention_dropout=0.0, |
| 241 | layernorm_epsilon=1e-5, |
| 242 | rmsnorm=True, |
| 243 | apply_residual_connection_post_layernorm=False, |
| 244 | post_layer_norm=True, |
| 245 | add_bias_linear=False, |
| 246 | add_qkv_bias=False, |
| 247 | bias_dropout_fusion=True, |
| 248 | multi_query_attention=False, |
| 249 | multi_query_group_num=1, |
| 250 | apply_query_key_layer_scaling=True, |
| 251 | attention_softmax_in_fp32=True, |
| 252 | fp32_residual_connection=False, |
| 253 | quantization_bit=0, |
| 254 | pre_seq_len=None, |
| 255 | prefix_projection=False, |
| 256 | **kwargs |
| 257 | ): |
| 258 | self.num_layers = num_layers |
| 259 | self.vocab_size = padded_vocab_size |
| 260 | self.padded_vocab_size = padded_vocab_size |
| 261 | self.hidden_size = hidden_size |
| 262 | self.ffn_hidden_size = ffn_hidden_size |
| 263 | self.kv_channels = kv_channels |
| 264 | self.num_attention_heads = num_attention_heads |
| 265 | self.seq_length = seq_length |
| 266 | self.hidden_dropout = hidden_dropout |
| 267 | self.classifier_dropout = classifier_dropout |
| 268 | self.attention_dropout = attention_dropout |
| 269 | self.layernorm_epsilon = layernorm_epsilon |
| 270 | self.rmsnorm = rmsnorm |
| 271 | self.apply_residual_connection_post_layernorm = apply_residual_connection_post_layernorm |
| 272 | self.post_layer_norm = post_layer_norm |
| 273 | self.add_bias_linear = add_bias_linear |
| 274 | self.add_qkv_bias = add_qkv_bias |
| 275 | self.bias_dropout_fusion = bias_dropout_fusion |
| 276 | self.multi_query_attention = multi_query_attention |
| 277 | self.multi_query_group_num = multi_query_group_num |
| 278 | self.apply_query_key_layer_scaling = apply_query_key_layer_scaling |
| 279 | self.attention_softmax_in_fp32 = attention_softmax_in_fp32 |
| 280 | self.fp32_residual_connection = fp32_residual_connection |
| 281 | self.quantization_bit = quantization_bit |
| 282 | self.pre_seq_len = pre_seq_len |
| 283 | self.prefix_projection = prefix_projection |
| 284 | super().__init__(**kwargs) |
| 285 | |
| 286 | |