| 123 | } |
| 124 | |
| 125 | [[noreturn]] |
| 126 | static void usage(const char * executable) { |
| 127 | printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--include-weights]\n", executable); |
| 128 | printf(" [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--tensor-type] [--tensor-type-file]\n"); |
| 129 | printf(" [--prune-layers] [--keep-split] [--override-kv] [--dry-run]\n"); |
| 130 | printf(" model-f32.gguf [model-quant.gguf] type [nthreads]\n\n"); |
| 131 | printf(" --allow-requantize\n"); |
| 132 | printf(" allow requantizing tensors that have already been quantized\n"); |
| 133 | printf(" WARNING: this can severely reduce quality compared to quantizing\n"); |
| 134 | printf(" from 16bit or 32bit!\n"); |
| 135 | printf(" --leave-output-tensor\n"); |
| 136 | printf(" leave output.weight un(re)quantized\n"); |
| 137 | printf(" increases model size but may also increase quality, especially when requantizing\n"); |
| 138 | printf(" --pure\n"); |
| 139 | printf(" disable k-quant mixtures and quantize all tensors to the same type\n"); |
| 140 | printf(" --imatrix file_name\n"); |
| 141 | printf(" use data in file_name as importance matrix for quant optimizations\n"); |
| 142 | printf(" --include-weights tensor_name\n"); |
| 143 | printf(" use importance matrix for this/these tensor(s)\n"); |
| 144 | printf(" --exclude-weights tensor_name\n"); |
| 145 | printf(" do not use importance matrix for this/these tensor(s)\n"); |
| 146 | printf(" --output-tensor-type ggml_type\n"); |
| 147 | printf(" use this ggml_type for the output.weight tensor\n"); |
| 148 | printf(" --token-embedding-type ggml_type\n"); |
| 149 | printf(" use this ggml_type for the token embeddings tensor\n"); |
| 150 | printf(" --tensor-type tensor_name=ggml_type\n"); |
| 151 | printf(" quantize this tensor to this ggml_type\n"); |
| 152 | printf(" this is an advanced option to selectively quantize tensors. may be specified multiple times.\n"); |
| 153 | printf(" example: --tensor-type attn_q=q8_0\n"); |
| 154 | printf(" --tensor-type-file tensor_types.txt\n"); |
| 155 | printf(" list of tensors to quantize to a specific ggml_type\n"); |
| 156 | printf(" this is an advanced option to selectively quantize a long list of tensors.\n"); |
| 157 | printf(" the file should use the same format as above, separated by spaces or newlines.\n"); |
| 158 | printf(" --prune-layers L0,L1,L2...\n"); |
| 159 | printf(" comma-separated list of layer numbers to prune from the model\n"); |
| 160 | printf(" WARNING: this is an advanced option, use with care.\n"); |
| 161 | printf(" --keep-split\n"); |
| 162 | printf(" generate quantized model in the same shards as input\n"); |
| 163 | printf(" --override-kv KEY=TYPE:VALUE\n"); |
| 164 | printf(" override model metadata by key in the quantized model. may be specified multiple times.\n"); |
| 165 | printf(" WARNING: this is an advanced option, use with care.\n"); |
| 166 | printf(" --dry-run\n"); |
| 167 | printf(" calculate and show the final quantization size without performing quantization\n"); |
| 168 | printf(" example: llama-quantize --dry-run model-f32.gguf Q4_K\n\n"); |
| 169 | printf("note: --include-weights and --exclude-weights cannot be used together\n\n"); |
| 170 | printf("-----------------------------------------------------------------------------\n"); |
| 171 | printf(" allowed quantization types\n"); |
| 172 | printf("-----------------------------------------------------------------------------\n\n"); |
| 173 | for (const auto & it : QUANT_OPTIONS) { |
| 174 | if (it.name != "COPY") { |
| 175 | printf(" %2d or ", it.ftype); |
| 176 | } else { |
| 177 | printf(" "); |
| 178 | } |
| 179 | printf("%-7s : %s\n", it.name.c_str(), it.desc.c_str()); |
| 180 | } |
| 181 | exit(1); |
| 182 | } |