(self, spec, module, quant_type=common_spec.Quantization.CT2)
| 2599 | spec.gamma = layer_norm.weight |
| 2600 | |
| 2601 | def set_decoder(self, spec, module, quant_type=common_spec.Quantization.CT2): |
| 2602 | spec.scale_embeddings = False |
| 2603 | self.set_embeddings(spec.embeddings, module.embed_tokens) |
| 2604 | self.set_layer_norm(spec.layer_norm, module.norm) |
| 2605 | |
| 2606 | for layer_spec, layer in zip(spec.layer, module.layers): |
| 2607 | self.set_layer_norm( |
| 2608 | layer_spec.self_attention.layer_norm, layer.input_layernorm |
| 2609 | ) |
| 2610 | self.set_layer_norm( |
| 2611 | layer_spec.ffn.layer_norm, layer.post_attention_layernorm |
| 2612 | ) |
| 2613 | |
| 2614 | split_layers = [common_spec.LinearSpec() for _ in range(3)] |
| 2615 | self.set_linear( |
| 2616 | split_layers[0], layer.self_attn.q_proj, quant_type=quant_type |
| 2617 | ) |
| 2618 | self.set_linear( |
| 2619 | split_layers[1], layer.self_attn.k_proj, quant_type=quant_type |
| 2620 | ) |
| 2621 | self.set_linear( |
| 2622 | split_layers[2], layer.self_attn.v_proj, quant_type=quant_type |
| 2623 | ) |
| 2624 | |
| 2625 | if quant_type == common_spec.Quantization.CT2: |
| 2626 | utils.fuse_linear(layer_spec.self_attention.linear[0], split_layers) |
| 2627 | else: |
| 2628 | cc_dim = 1 if quant_type == common_spec.Quantization.AWQ_GEMM else 0 |
| 2629 | utils.fuse_linear_prequant( |
| 2630 | layer_spec.self_attention.linear[0], split_layers, cc_dim |
| 2631 | ) |
| 2632 | |
| 2633 | self.set_linear( |
| 2634 | layer_spec.self_attention.linear[1], |
| 2635 | layer.self_attn.o_proj, |
| 2636 | quant_type=quant_type, |
| 2637 | ) |
| 2638 | |
| 2639 | self.set_linear( |
| 2640 | layer_spec.ffn.linear_0, layer.mlp.gate_proj, quant_type=quant_type |
| 2641 | ) |
| 2642 | self.set_linear( |
| 2643 | layer_spec.ffn.linear_0_noact, layer.mlp.up_proj, quant_type=quant_type |
| 2644 | ) |
| 2645 | self.set_linear( |
| 2646 | layer_spec.ffn.linear_1, layer.mlp.down_proj, quant_type=quant_type |
| 2647 | ) |
| 2648 | |
| 2649 | delattr(layer, "self_attn") |
| 2650 | delattr(layer, "mlp") |
| 2651 | gc.collect() |
| 2652 | |
| 2653 | |
| 2654 | @register_loader("Qwen3Config") |
no test coverage detected