(weight: torch.Tensor, scale_list: torch.Tensor, source_bit_width: int)
| 56 | |
| 57 | |
| 58 | def extract_weight_to_half(weight: torch.Tensor, scale_list: torch.Tensor, source_bit_width: int): |
| 59 | if source_bit_width == 8: |
| 60 | func = kernels.int8WeightExtractionHalf |
| 61 | elif source_bit_width == 4: |
| 62 | func = kernels.int4WeightExtractionHalf |
| 63 | else: |
| 64 | assert False, "Unsupported bit-width" |
| 65 | |
| 66 | with torch.cuda.device(weight.device): |
| 67 | n, m = weight.size(0), weight.size(1) |
| 68 | out = torch.empty(n, m * (8 // source_bit_width), dtype=torch.half, device="cuda") |
| 69 | stream = torch.cuda.current_stream() |
| 70 | |
| 71 | gridDim = (n, 1, 1) |
| 72 | blockDim = (min(round_up(m, 32), 1024), 1, 1) |
| 73 | |
| 74 | func( |
| 75 | gridDim, |
| 76 | blockDim, |
| 77 | 0, |
| 78 | stream, |
| 79 | [ |
| 80 | ctypes.c_void_p(weight.data_ptr()), |
| 81 | ctypes.c_void_p(scale_list.data_ptr()), |
| 82 | ctypes.c_void_p(out.data_ptr()), |
| 83 | ctypes.c_int32(n), |
| 84 | ctypes.c_int32(m), |
| 85 | ], |
| 86 | ) |
| 87 | return out |
| 88 | |
| 89 | |
| 90 | if __name__ == "__main__": |
no test coverage detected