(self, spec, module)
| 1560 | spec.layer_norm_use_residual = True |
| 1561 | |
| 1562 | def set_decoder(self, spec, module): |
| 1563 | spec.scale_embeddings = True |
| 1564 | spec.start_from_zero_embedding = False |
| 1565 | self.set_embeddings(spec.embeddings, module.embed_tokens) |
| 1566 | self.set_layer_norm(spec.layer_norm, module.norm) |
| 1567 | |
| 1568 | for layer_spec, layer in zip(spec.layer, module.layers): |
| 1569 | self.set_layer_norm( |
| 1570 | layer_spec.self_attention.layer_norm, layer.input_layernorm |
| 1571 | ) |
| 1572 | self.set_layer_norm( |
| 1573 | layer_spec.ffn.layer_norm, layer.post_attention_layernorm |
| 1574 | ) |
| 1575 | |
| 1576 | wq = layer.self_attn.q_proj.weight |
| 1577 | wk = layer.self_attn.k_proj.weight |
| 1578 | wv = layer.self_attn.v_proj.weight |
| 1579 | wo = layer.self_attn.o_proj.weight |
| 1580 | |
| 1581 | layer_spec.self_attention.linear[0].weight = torch.cat([wq, wk, wv]) |
| 1582 | layer_spec.self_attention.linear[1].weight = wo |
| 1583 | |
| 1584 | self.set_linear(layer_spec.ffn.linear_0, layer.mlp.gate_proj) |
| 1585 | self.set_linear(layer_spec.ffn.linear_0_noact, layer.mlp.up_proj) |
| 1586 | self.set_linear(layer_spec.ffn.linear_1, layer.mlp.down_proj) |
| 1587 | |
| 1588 | delattr(layer, "self_attn") |
| 1589 | delattr(layer, "mlp") |
| 1590 | gc.collect() |
| 1591 | |
| 1592 | |
| 1593 | @register_loader("Gemma2Config") |
no test coverage detected