(
model, w_bit, q_config, quant_type="int"
)
| 64 | |
| 65 | @torch.no_grad() |
| 66 | def pseudo_quantize_model_weight( |
| 67 | model, w_bit, q_config, quant_type="int" |
| 68 | ): |
| 69 | if quant_type == "int": |
| 70 | layers = model.model.layers |
| 71 | for i in tqdm(range(len(layers)), desc=f"pseudo {quant_type} weight quantization..."): |
| 72 | named_linears = get_named_linears(layers[i]) |
| 73 | for n, m in named_linears.items(): |
| 74 | m.weight.data = pseudo_quantize_tensor(m.weight.data, n_bit=w_bit, **q_config) |
| 75 | |
| 76 | elif quant_type == "nf3": |
| 77 | quantizer = SteN2F3Quantizer(q_group_size=q_config["q_group_size"]) |
| 78 | layers = model.model.layers |
| 79 | for i in tqdm(range(len(layers)), desc=f"pseudo {quant_type} weight quantization..."): |
| 80 | named_linears = get_named_linears(layers[i]) |
| 81 | for n, m in named_linears.items(): |
| 82 | # m.cuda() |
| 83 | m.weight.data = quantizer(m.weight.data) |
no test coverage detected