(weight: torch.Tensor)
| 35 | |
| 36 | |
| 37 | def compress_int4_weight(weight: torch.Tensor): # (n, m) |
| 38 | with torch.cuda.device(weight.device): |
| 39 | n, m = weight.size(0), weight.size(1) |
| 40 | assert m % 2 == 0 |
| 41 | m = m // 2 |
| 42 | out = torch.empty(n, m, dtype=torch.int8, device="cuda") |
| 43 | stream = torch.cuda.current_stream() |
| 44 | |
| 45 | gridDim = (n, 1, 1) |
| 46 | blockDim = (min(round_up(m, 32), 1024), 1, 1) |
| 47 | |
| 48 | kernels.int4WeightCompression( |
| 49 | gridDim, |
| 50 | blockDim, |
| 51 | 0, |
| 52 | stream, |
| 53 | [ctypes.c_void_p(weight.data_ptr()), ctypes.c_void_p(out.data_ptr()), ctypes.c_int32(n), ctypes.c_int32(m)], |
| 54 | ) |
| 55 | return out |
| 56 | |
| 57 | |
| 58 | def extract_weight_to_half(weight: torch.Tensor, scale_list: torch.Tensor, source_bit_width: int): |
no test coverage detected