(weight_and_scale_names: List[Tuple[str, str]])
| 342 | return quantize(t) |
| 343 | |
| 344 | def conv_experts(weight_and_scale_names: List[Tuple[str, str]]) -> Tuple[torch.Tensor, torch.Tensor]: |
| 345 | nonlocal progress |
| 346 | progress += 1 |
| 347 | expert_weights = [weights[weight_name] for weight_name, _ in weight_and_scale_names] |
| 348 | if weight_and_scale_names[0][1] in weights: |
| 349 | for i in range(len(weight_and_scale_names)): |
| 350 | scale = weights[weight_and_scale_names[i][1]] |
| 351 | expert_weights[i] = blockwise_dequantize(expert_weights[i], scale, dequant_block_size) |
| 352 | t = torch.stack(expert_weights) |
| 353 | print(f"\rConverting tensor {progress}: {t.shape}", end="", flush=True) |
| 354 | if dtype not in [torch.float32, torch.float16]: |
| 355 | if isinstance(metadata.quant, KQuant): |
| 356 | t = per_expert_k_quantize(t.to(torch.float32), metadata.quant.name) |
| 357 | elif metadata.quant.block_size is None: |
| 358 | return per_tensor_quantize(t, dtype) |
| 359 | else: |
| 360 | quant_block_size = torch.tensor(metadata.quant.block_size) |
| 361 | return per_expert_blockwise_quantize(t, quant_block_size, dtype) |
| 362 | return t.to(dtype), None |
| 363 | |
| 364 | def save_weight_and_scale(weight_name: str, scale_name: str, weight_and_scale: Tuple[torch.Tensor, torch.Tensor]): |
| 365 | tensors[weight_name] = weight_and_scale[0] |
no test coverage detected