| 288 | Block::~Block() {} |
| 289 | |
| 290 | void Block::block( |
| 291 | InferenceState& s, |
| 292 | int pos, |
| 293 | int kv_sink, |
| 294 | int kv_pos, |
| 295 | int kv_len |
| 296 | ) const { |
| 297 | if (_device == Device::CPU) { |
| 298 | switch (_config->weight_quant) { |
| 299 | case Quant::F32: |
| 300 | _block_cpu<float>(s, pos, kv_sink, kv_pos, kv_len); |
| 301 | break; |
| 302 | case Quant::F16: |
| 303 | #if defined(__AVX2__) && defined(__F16C__) |
| 304 | _block_cpu<f16_t>(s, pos, kv_sink, kv_pos, kv_len); |
| 305 | #else |
| 306 | assert(false && "float16 not supported on this platform"); |
| 307 | #endif |
| 308 | break; |
| 309 | case Quant::F8E5M2: |
| 310 | _block_cpu<f8e5m2_t>(s, pos, kv_sink, kv_pos, kv_len); |
| 311 | break; |
| 312 | case Quant::Q2_K: |
| 313 | _block_cpu<block_q2_K>(s, pos, kv_sink, kv_pos, kv_len); |
| 314 | break; |
| 315 | case Quant::Q3_K: |
| 316 | _block_cpu<block_q3_K>(s, pos, kv_sink, kv_pos, kv_len); |
| 317 | break; |
| 318 | default: |
| 319 | assert(false && "unsupported weight quantization for cpu"); |
| 320 | } |
| 321 | } |
| 322 | } |
| 323 | |
| 324 | double Block::active_bytes(size_t pos) const { |
| 325 | double bytes_per_weight = bits_per_weight(_config->weight_quant, _config->block_size[0] * _config->block_size[1]) / 8.0; |