Function make_quant_attn

inference/modules/fused_attn.py:304–370

Replace all LlamaAttention modules with QuantLlamaAttention modules, fusing the q, k, v projections.

(model, dev)

def make_quant_attn(model, dev):
    """
    Replace all LlamaAttention modules with QuantLlamaAttention modules, fusing the q, k, v projections.
    """
    model = model.cpu()
    for name, m in model.named_modules():
        if not m.__class__.__name__ in ["LlamaAttention", "LlamaAttentionFused"]:
            continue

        q_proj = m.q_proj
        k_proj = m.k_proj
        v_proj = m.v_proj

        qweights = torch.cat([q_proj.qweight, k_proj.qweight, v_proj.qweight], dim=0)
        qzeros = torch.cat([q_proj.qzeros, k_proj.qzeros, v_proj.qzeros], dim=0)
        scales = torch.cat([q_proj.scales, k_proj.scales, v_proj.scales], dim=0)
        # g_idx = torch.cat([q_proj.g_idx, k_proj.g_idx, v_proj.g_idx], dim=0)
        g_idx = None
        bias = (
            torch.cat([q_proj.bias, k_proj.bias, v_proj.bias], dim=0)
            if q_proj.bias is not None
            else None
        )

        qkv_layer = WQLinear(
            q_proj.w_bit,
            q_proj.group_size,
            q_proj.in_features,
            q_proj.out_features + k_proj.out_features + v_proj.out_features,
            q_proj.bias is not None,
            q_proj.qweight.device,
        )
        qkv_layer.qweight = qweights
        qkv_layer.qzeros = qzeros
        qkv_layer.scales = scales

        qkv_layer.bias = bias
        qkv_layer.split_k_iters = q_proj.split_k_iters
        # We're dropping the rotary embedding layer m.rotary_emb here. We don't need it in the triton branch.

        if isinstance(m, LlamaAttention):
            attn = QuantLlamaAttention(
                m.hidden_size, m.num_heads, qkv_layer, m.o_proj, dev
            )
        else:
            attn = QuantLlamaAttentionFused(
                m.args.hidden_size,
                m.args.num_attention_heads,
                qkv_layer,
                m.o_proj,
                dev,
                m.args,
            )
        if "." in name:
            parent_name = name.rsplit(".", 1)[0]
            child_name = name[len(parent_name) + 1 :]
            parent = model.get_submodule(parent_name)
        else: