| 38 | } |
| 39 | |
| 40 | double bits_per_weight(Quant quant, size_t blockwise_quant_size) { |
| 41 | if (blockwise_quant_size > 0 && quant != Quant::F8E5M2) { |
| 42 | std::cerr << "blockwise quantization should only be used with F8E5M2" << std::endl; |
| 43 | assert(false); |
| 44 | } |
| 45 | switch (quant) { |
| 46 | case Quant::F32: return 32; |
| 47 | case Quant::F16: return 16; |
| 48 | case Quant::F8E5M2: return (8 + blockwise_quant_size) / blockwise_quant_size; |
| 49 | case Quant::Q2_K: return 2.5625; |
| 50 | case Quant::Q3_K: return 3.4375; |
| 51 | } |
| 52 | __builtin_unreachable(); |
| 53 | } |
| 54 | |
| 55 | CodecDType quant_to_codec_dtype(Quant quant) { |
| 56 | switch (quant) { |