| 1808 | // ====================== 4-bit (de)-quantization |
| 1809 | |
| 1810 | void quantize_row_q4_K_reference(const float * restrict x, block_q4_K * restrict y, int k) { |
| 1811 | assert(k % QK_K == 0); |
| 1812 | const int nb = k / QK_K; |
| 1813 | |
| 1814 | uint8_t L[QK_K]; |
| 1815 | uint8_t Laux[32]; |
| 1816 | float weights[32]; |
| 1817 | float mins[QK_K/32]; |
| 1818 | float scales[QK_K/32]; |
| 1819 | |
| 1820 | for (int i = 0; i < nb; i++) { |
| 1821 | |
| 1822 | float max_scale = 0; // as we are deducting the min, scales are always positive |
| 1823 | float max_min = 0; |
| 1824 | for (int j = 0; j < QK_K/32; ++j) { |
| 1825 | //scales[j] = make_qkx1_quants(32, 15, x + 32*j, L + 32*j, &mins[j], 9, 0.5f); |
| 1826 | float sum_x2 = 0; |
| 1827 | for (int l = 0; l < 32; ++l) sum_x2 += x[32*j + l] * x[32*j + l]; |
| 1828 | float av_x = sqrtf(sum_x2/32); |
| 1829 | for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]); |
| 1830 | scales[j] = make_qkx2_quants(32, 15, x + 32*j, weights, L + 32*j, &mins[j], Laux, -1.f, 0.1f, 20, false); |
| 1831 | float scale = scales[j]; |
| 1832 | if (scale > max_scale) { |
| 1833 | max_scale = scale; |
| 1834 | } |
| 1835 | float min = mins[j]; |
| 1836 | if (min > max_min) { |
| 1837 | max_min = min; |
| 1838 | } |
| 1839 | } |
| 1840 | |
| 1841 | #if QK_K == 256 |
| 1842 | float inv_scale = max_scale > 0 ? 63.f/max_scale : 0.f; |
| 1843 | float inv_min = max_min > 0 ? 63.f/max_min : 0.f; |
| 1844 | for (int j = 0; j < QK_K/32; ++j) { |
| 1845 | uint8_t ls = nearest_int(inv_scale*scales[j]); |
| 1846 | uint8_t lm = nearest_int(inv_min*mins[j]); |
| 1847 | ls = MIN(63, ls); |
| 1848 | lm = MIN(63, lm); |
| 1849 | if (j < 4) { |
| 1850 | y[i].scales[j] = ls; |
| 1851 | y[i].scales[j+4] = lm; |
| 1852 | } else { |
| 1853 | y[i].scales[j+4] = (ls & 0xF) | ((lm & 0xF) << 4); |
| 1854 | y[i].scales[j-4] |= ((ls >> 4) << 6); |
| 1855 | y[i].scales[j-0] |= ((lm >> 4) << 6); |
| 1856 | } |
| 1857 | } |
| 1858 | y[i].d = GGML_FP32_TO_FP16(max_scale/63.f); |
| 1859 | y[i].dmin = GGML_FP32_TO_FP16(max_min/63.f); |
| 1860 | |
| 1861 | uint8_t sc, m; |
| 1862 | for (int j = 0; j < QK_K/32; ++j) { |
| 1863 | get_scale_min_k4(j, y[i].scales, &sc, &m); |
| 1864 | const float d = GGML_FP16_TO_FP32(y[i].d) * sc; |
| 1865 | if (!d) continue; |
| 1866 | const float dm = GGML_FP16_TO_FP32(y[i].dmin) * m; |
| 1867 | for (int ii = 0; ii < 32; ++ii) { |
no test coverage detected