| 55 | }; |
| 56 | |
| 57 | static void llama_tensor_dequantize_impl( |
| 58 | ggml_tensor * tensor, std::vector<no_init<float>> & output, std::vector<std::thread> & workers, |
| 59 | const size_t nelements, const int nthread |
| 60 | ) { |
| 61 | if (output.size() < nelements) { |
| 62 | output.resize(nelements); |
| 63 | } |
| 64 | float * f32_output = (float *) output.data(); |
| 65 | |
| 66 | const ggml_type_traits * qtype = ggml_get_type_traits(tensor->type); |
| 67 | if (ggml_is_quantized(tensor->type)) { |
| 68 | if (qtype->to_float == NULL) { |
| 69 | throw std::runtime_error(format("type %s unsupported for integer quantization: no dequantization available", ggml_type_name(tensor->type))); |
| 70 | } |
| 71 | } else if (tensor->type != GGML_TYPE_F16 && |
| 72 | tensor->type != GGML_TYPE_BF16) { |
| 73 | throw std::runtime_error(format("cannot dequantize/convert tensor type %s", ggml_type_name(tensor->type))); |
| 74 | } |
| 75 | |
| 76 | if (nthread < 2) { |
| 77 | if (tensor->type == GGML_TYPE_F16) { |
| 78 | ggml_fp16_to_fp32_row((ggml_fp16_t *)tensor->data, f32_output, nelements); |
| 79 | } else if (tensor->type == GGML_TYPE_BF16) { |
| 80 | ggml_bf16_to_fp32_row((ggml_bf16_t *)tensor->data, f32_output, nelements); |
| 81 | } else if (ggml_is_quantized(tensor->type)) { |
| 82 | qtype->to_float(tensor->data, f32_output, nelements); |
| 83 | } else { |
| 84 | GGML_ABORT("fatal error"); // unreachable |
| 85 | } |
| 86 | return; |
| 87 | } |
| 88 | |
| 89 | size_t block_size; |
| 90 | if (tensor->type == GGML_TYPE_F16 || |
| 91 | tensor->type == GGML_TYPE_BF16) { |
| 92 | block_size = 1; |
| 93 | } else { |
| 94 | block_size = (size_t)ggml_blck_size(tensor->type); |
| 95 | } |
| 96 | |
| 97 | size_t block_size_bytes = ggml_type_size(tensor->type); |
| 98 | |
| 99 | GGML_ASSERT(nelements % block_size == 0); |
| 100 | size_t nblocks = nelements / block_size; |
| 101 | size_t blocks_per_thread = nblocks / nthread; |
| 102 | size_t spare_blocks = nblocks - (blocks_per_thread * nthread); // if blocks aren't divisible by thread count |
| 103 | |
| 104 | size_t in_buff_offs = 0; |
| 105 | size_t out_buff_offs = 0; |
| 106 | |
| 107 | for (int tnum = 0; tnum < nthread; tnum++) { |
| 108 | size_t thr_blocks = blocks_per_thread + (tnum == nthread - 1 ? spare_blocks : 0); // num blocks for this thread |
| 109 | size_t thr_elems = thr_blocks * block_size; // number of elements for this thread |
| 110 | size_t thr_block_bytes = thr_blocks * block_size_bytes; // number of input bytes for this thread |
| 111 | |
| 112 | auto compute = [qtype] (ggml_type typ, uint8_t * inbuf, float * outbuf, int nels) { |
| 113 | if (typ == GGML_TYPE_F16) { |
| 114 | ggml_fp16_to_fp32_row((ggml_fp16_t *)inbuf, outbuf, nels); |
no test coverage detected