| 491 | } |
| 492 | |
| 493 | void BlockMHA::attention_impl( |
| 494 | InferenceState& s, int pos, int kv_sink, int kv_pos, int kv_len |
| 495 | ) const { |
| 496 | switch (_config->weight_quant) { |
| 497 | case Quant::F32: |
| 498 | _attention_impl<float>(s, pos, kv_sink, kv_pos, kv_len); |
| 499 | break; |
| 500 | case Quant::F16: |
| 501 | _attention_impl<f16_t>(s, pos, kv_sink, kv_pos, kv_len); |
| 502 | break; |
| 503 | case Quant::F8E5M2: |
| 504 | _attention_impl<f8e5m2_t>(s, pos, kv_sink, kv_pos, kv_len); |
| 505 | break; |
| 506 | case Quant::Q2_K: |
| 507 | _attention_impl<block_q2_K>(s, pos, kv_sink, kv_pos, kv_len); |
| 508 | break; |
| 509 | case Quant::Q3_K: |
| 510 | _attention_impl<block_q3_K>(s, pos, kv_sink, kv_pos, kv_len); |
| 511 | break; |
| 512 | default: |
| 513 | assert(false && "unsupported weight quantization for mha"); |
| 514 | } |
| 515 | } |
| 516 | |
| 517 | |
| 518 | BlockMLA::BlockMLA( |
nothing calls this directly
no outgoing calls
no test coverage detected