(self, spec, module)
| 1657 | spec.layer_norm_use_residual = True |
| 1658 | |
| 1659 | def set_decoder(self, spec, module): |
| 1660 | spec.scale_embeddings = True |
| 1661 | spec.start_from_zero_embedding = False |
| 1662 | self.set_embeddings(spec.embeddings, module.embed_tokens) |
| 1663 | self.set_layer_norm(spec.layer_norm, module.norm) |
| 1664 | |
| 1665 | for layer_spec, layer in zip(spec.layer, module.layers): |
| 1666 | self.set_layer_norm(layer_spec.input_layer_norm, layer.input_layernorm) |
| 1667 | |
| 1668 | self.set_layer_norm( |
| 1669 | layer_spec.post_attention_layer_norm, layer.post_attention_layernorm |
| 1670 | ) |
| 1671 | |
| 1672 | self.set_layer_norm( |
| 1673 | layer_spec.pre_feedforward_layer_norm, layer.pre_feedforward_layernorm |
| 1674 | ) |
| 1675 | |
| 1676 | self.set_layer_norm( |
| 1677 | layer_spec.post_feedforward_layer_norm, layer.post_feedforward_layernorm |
| 1678 | ) |
| 1679 | |
| 1680 | wq = layer.self_attn.q_proj.weight |
| 1681 | wk = layer.self_attn.k_proj.weight |
| 1682 | wv = layer.self_attn.v_proj.weight |
| 1683 | wo = layer.self_attn.o_proj.weight |
| 1684 | |
| 1685 | layer_spec.self_attention.linear[0].weight = torch.cat([wq, wk, wv]) |
| 1686 | layer_spec.self_attention.linear[1].weight = wo |
| 1687 | |
| 1688 | self.set_linear(layer_spec.ffn.linear_0, layer.mlp.gate_proj) |
| 1689 | self.set_linear(layer_spec.ffn.linear_0_noact, layer.mlp.up_proj) |
| 1690 | self.set_linear(layer_spec.ffn.linear_1, layer.mlp.down_proj) |
| 1691 | |
| 1692 | delattr(layer, "self_attn") |
| 1693 | delattr(layer, "mlp") |
| 1694 | gc.collect() |
| 1695 | |
| 1696 | |
| 1697 | @register_loader("LlamaConfig") |
no test coverage detected