| 68 | |
| 69 | |
| 70 | def compress_int4_weight(weight: torch.Tensor): # (n, m) |
| 71 | with torch.cuda.device(weight.device): |
| 72 | n, m = weight.size(0), weight.size(1) |
| 73 | assert m % 2 == 0 |
| 74 | m = m // 2 |
| 75 | out = torch.empty(n, m, dtype=torch.int8, device="cuda") |
| 76 | stream = torch.cuda.current_stream() |
| 77 | |
| 78 | gridDim = (n, 1, 1) |
| 79 | blockDim = (min(round_up(m, 32), 1024), 1, 1) |
| 80 | |
| 81 | kernels.int4WeightCompression( |
| 82 | gridDim, |
| 83 | blockDim, |
| 84 | 0, |
| 85 | stream, |
| 86 | [ctypes.c_void_p(weight.data_ptr()), ctypes.c_void_p(out.data_ptr()), ctypes.c_int32(n), ctypes.c_int32(m)], |
| 87 | ) |
| 88 | return out |
| 89 | |
| 90 | |
| 91 | def extract_weight_to_half(weight: torch.Tensor, scale_list: torch.Tensor, source_bit_width: int): |