Function quantize_with_gptq

models/quantization.py:385–393

(model, wbits, groupsize)

def quantize_with_gptq(model, wbits, groupsize):
    model = model.eval()
    layers = find_layers(model)
    for name in ['lm_head']:
        if name in layers:
            del layers[name]
    make_quant(model, layers, wbits, groupsize)
    # model.load_state_dict(torch.load(checkpoint))
    return model