Verify every tensor type and dimension used by the specialized pipeline. * For distributed sliced GGUFs, only the advertised local layer range is * required; token embedding and output head are validated when present. */
| 3544 | for (int j = 0; j < QK_K / 32; j++) { |
| 3545 | uint8_t sc_val, m_val; |
| 3546 | q4_k_get_scale_min(j, sc, &sc_val, &m_val); |
| 3547 | int32_t gsum = (int32_t)y[i].bsums[j * 2] + (int32_t)y[i].bsums[j * 2 + 1]; |
| 3548 | summs += m_val * gsum; |
| 3549 | } |
| 3550 | |
| 3551 | int isum = 0; |
| 3552 | for (int j = 0; j < QK_K / 32; j++) { |
| 3553 | uint8_t sc_val, m_val; |
| 3554 | q4_k_get_scale_min(j, sc, &sc_val, &m_val); |
| 3555 | |
| 3556 | const int byte_off = (j >> 1) * 32; |
| 3557 | const int shift = (j & 1) * 4; |
| 3558 | |
| 3559 | /* Load 32 q8 values for this group */ |
| 3560 | const int8x16x2_t q8v = vld1q_s8_x2(q8 + j * 32); |
| 3561 | |
| 3562 | /* Unpack 32 q4 values from 32 bytes at qs[byte_off] with shift */ |
| 3563 | uint8_t q4_u[32]; |
| 3564 | if (shift == 0) { |
| 3565 | for (int l = 0; l < 32; l++) q4_u[l] = qs[byte_off + l] & 0xF; |
| 3566 | } else { |
| 3567 | for (int l = 0; l < 32; l++) q4_u[l] = qs[byte_off + l] >> 4; |
| 3568 | } |
| 3569 | |
| 3570 | const int8x16_t q4a = vreinterpretq_s8_u8(vld1q_u8(q4_u)); |
| 3571 | const int8x16_t q4b = vreinterpretq_s8_u8(vld1q_u8(q4_u + 16)); |
| 3572 | |
| 3573 | isum += vaddvq_s32(vdotq_s32(zero, q4a, q8v.val[0])) * sc_val; |
| 3574 | isum += vaddvq_s32(vdotq_s32(zero, q4b, q8v.val[1])) * sc_val; |
| 3575 | } |
| 3576 | |
| 3577 | sumf += d * (float)isum + dm * (float)summs; |
| 3578 | } |
| 3579 | |
| 3580 | *s = sumf; |
| 3581 | #else |
| 3582 | float sumf = 0.0f; |
| 3583 | |
| 3584 | for (int i = 0; i < nb; i++) { |
| 3585 | const float d = y[i].d * f16_to_f32(x[i].d); |
| 3586 | const float dm = -y[i].d * f16_to_f32(x[i].dmin); |
| 3587 | |
| 3588 | const uint8_t *qs = x[i].qs; |
| 3589 | const uint8_t *sc = x[i].scales; |
| 3590 | const int8_t *q8 = y[i].qs; |
| 3591 | |
| 3592 | int summs = 0; |
| 3593 | for (int j = 0; j < QK_K / 32; j++) { |
| 3594 | uint8_t sc_val, m_val; |
| 3595 | q4_k_get_scale_min(j, sc, &sc_val, &m_val); |
| 3596 | int32_t gsum = (int32_t)y[i].bsums[j * 2] + (int32_t)y[i].bsums[j * 2 + 1]; |
| 3597 | summs += m_val * gsum; |
| 3598 | } |
| 3599 | |
| 3600 | int isum = 0; |
| 3601 | for (int j = 0; j < QK_K / 32; j++) { |
| 3602 | uint8_t sc_val, m_val; |
| 3603 | q4_k_get_scale_min(j, sc, &sc_val, &m_val); |
no test coverage detected