| 471 | } |
| 472 | |
| 473 | static void llama_model_quantize_impl(const std::string & fname_inp, const std::string & fname_out, const llama_model_quantize_params * params) { |
| 474 | ggml_type default_type; |
| 475 | llama_ftype ftype = params->ftype; |
| 476 | |
| 477 | switch (params->ftype) { |
| 478 | case LLAMA_FTYPE_MOSTLY_Q4_0: default_type = GGML_TYPE_Q4_0; break; |
| 479 | case LLAMA_FTYPE_MOSTLY_Q4_1: default_type = GGML_TYPE_Q4_1; break; |
| 480 | case LLAMA_FTYPE_MOSTLY_Q5_0: default_type = GGML_TYPE_Q5_0; break; |
| 481 | case LLAMA_FTYPE_MOSTLY_Q5_1: default_type = GGML_TYPE_Q5_1; break; |
| 482 | case LLAMA_FTYPE_MOSTLY_Q8_0: default_type = GGML_TYPE_Q8_0; break; |
| 483 | case LLAMA_FTYPE_MOSTLY_F16: default_type = GGML_TYPE_F16; break; |
| 484 | case LLAMA_FTYPE_MOSTLY_BF16: default_type = GGML_TYPE_BF16; break; |
| 485 | case LLAMA_FTYPE_ALL_F32: default_type = GGML_TYPE_F32; break; |
| 486 | |
| 487 | // K-quants |
| 488 | case LLAMA_FTYPE_MOSTLY_Q2_K_S: |
| 489 | case LLAMA_FTYPE_MOSTLY_Q2_K: default_type = GGML_TYPE_Q2_K; break; |
| 490 | case LLAMA_FTYPE_MOSTLY_IQ3_XS: default_type = GGML_TYPE_IQ3_S; break; |
| 491 | case LLAMA_FTYPE_MOSTLY_Q3_K_S: |
| 492 | case LLAMA_FTYPE_MOSTLY_Q3_K_M: |
| 493 | case LLAMA_FTYPE_MOSTLY_Q3_K_L: default_type = GGML_TYPE_Q3_K; break; |
| 494 | case LLAMA_FTYPE_MOSTLY_Q4_K_S: |
| 495 | case LLAMA_FTYPE_MOSTLY_Q4_K_M: default_type = GGML_TYPE_Q4_K; break; |
| 496 | case LLAMA_FTYPE_MOSTLY_Q5_K_S: |
| 497 | case LLAMA_FTYPE_MOSTLY_Q5_K_M: default_type = GGML_TYPE_Q5_K; break; |
| 498 | case LLAMA_FTYPE_MOSTLY_Q6_K: default_type = GGML_TYPE_Q6_K; break; |
| 499 | case LLAMA_FTYPE_MOSTLY_TQ1_0: default_type = GGML_TYPE_TQ1_0; break; |
| 500 | case LLAMA_FTYPE_MOSTLY_TQ2_0: default_type = GGML_TYPE_TQ2_0; break; |
| 501 | case LLAMA_FTYPE_MOSTLY_IQ2_XXS: default_type = GGML_TYPE_IQ2_XXS; break; |
| 502 | case LLAMA_FTYPE_MOSTLY_IQ2_XS: default_type = GGML_TYPE_IQ2_XS; break; |
| 503 | case LLAMA_FTYPE_MOSTLY_IQ2_S: default_type = GGML_TYPE_IQ2_XS; break; |
| 504 | case LLAMA_FTYPE_MOSTLY_IQ2_M: default_type = GGML_TYPE_IQ2_S; break; |
| 505 | case LLAMA_FTYPE_MOSTLY_IQ3_XXS: default_type = GGML_TYPE_IQ3_XXS; break; |
| 506 | case LLAMA_FTYPE_MOSTLY_IQ1_S: default_type = GGML_TYPE_IQ1_S; break; |
| 507 | case LLAMA_FTYPE_MOSTLY_IQ1_M: default_type = GGML_TYPE_IQ1_M; break; |
| 508 | case LLAMA_FTYPE_MOSTLY_IQ4_NL: default_type = GGML_TYPE_IQ4_NL; break; |
| 509 | case LLAMA_FTYPE_MOSTLY_IQ4_XS: default_type = GGML_TYPE_IQ4_XS; break; |
| 510 | case LLAMA_FTYPE_MOSTLY_IQ3_S: default_type = GGML_TYPE_IQ3_S; break; |
| 511 | case LLAMA_FTYPE_MOSTLY_IQ3_M: default_type = GGML_TYPE_IQ3_S; break; |
| 512 | |
| 513 | default: throw std::runtime_error(format("invalid output file type %d\n", ftype)); |
| 514 | } |
| 515 | |
| 516 | int nthread = params->nthread; |
| 517 | |
| 518 | if (nthread <= 0) { |
| 519 | nthread = std::thread::hardware_concurrency(); |
| 520 | } |
| 521 | |
| 522 | // mmap consistently increases speed on Linux, and also increases speed on Windows with |
| 523 | // hot cache. It may cause a slowdown on macOS, possibly related to free memory. |
| 524 | #if defined(__linux__) || defined(_WIN32) |
| 525 | constexpr bool use_mmap = true; |
| 526 | #else |
| 527 | constexpr bool use_mmap = false; |
| 528 | #endif |
| 529 | |
| 530 | llama_model_kv_override * kv_overrides = nullptr; |
no test coverage detected