| 475 | for (size_t i = 0; i < absmax.size(); ++i) { |
| 476 | scales[i] = nested_quant_map[absmax[i]] * |
| 477 | nested_absmax[static_cast<size_t>(static_cast<int64_t>(i) / state.nested_blocksize)] + |
| 478 | state.nested_offset; |
| 479 | } |
| 480 | const auto packed_values = raw_u8_values(packed); |
| 481 | if (static_cast<int64_t>(packed_values.size()) != (elements + 1) / 2) { |
| 482 | throw std::runtime_error("BNB NF4 packed byte count mismatch: " + metadata.name); |
| 483 | } |
| 484 | const int64_t block_count = (elements + state.blocksize - 1) / state.blocksize; |
| 485 | if (static_cast<int64_t>(scales.size()) != block_count) { |
| 486 | throw std::runtime_error("BNB NF4 absmax block count mismatch: " + metadata.name); |
| 487 | } |
| 488 | std::vector<float> values(static_cast<size_t>(elements)); |
| 489 | int64_t out = 0; |
| 490 | for (const uint8_t byte : packed_values) { |
| 491 | values[static_cast<size_t>(out)] = |
| 492 | quant_map[(byte >> 4U) & 0x0FU] * scales[static_cast<size_t>(out / state.blocksize)]; |
| 493 | ++out; |
| 494 | if (out < elements) { |
| 495 | values[static_cast<size_t>(out)] = |
| 496 | quant_map[byte & 0x0FU] * scales[static_cast<size_t>(out / state.blocksize)]; |
| 497 | ++out; |
| 498 | } |
| 499 | } |
| 500 | RawTensorData out_data; |
| 501 | out_data.metadata = {metadata.name, dtype_for_ggml_type(output_type), state.shape}; |
| 502 | out_data.bytes = encode_f32_tensor_data(metadata.name, values, shape_from_dims(state.shape), output_type); |
no test coverage detected