Extract a `Config` from GGUF metadata.
(
path: &Path,
)
| 76 | |
| 77 | /// Extract a `Config` from GGUF metadata. |
| 78 | pub fn config_from_gguf( |
| 79 | path: &Path, |
| 80 | ) -> anyhow::Result<crate::models::common::Config> { |
| 81 | use candle_core::quantized::gguf_file; |
| 82 | |
| 83 | let mut file = std::fs::File::open(path)?; |
| 84 | let content = gguf_file::Content::read(&mut file) |
| 85 | .map_err(|e| anyhow::anyhow!("failed to read GGUF: {e}"))?; |
| 86 | |
| 87 | let arch = get_str(&content, "general.architecture") |
| 88 | .unwrap_or_else(|| "llama".to_string()); |
| 89 | |
| 90 | let hidden_size = get_u32(&content, &format!("{arch}.embedding_length")).unwrap_or(4096) as usize; |
| 91 | let intermediate_size = |
| 92 | get_u32(&content, &format!("{arch}.feed_forward_length")).unwrap_or(11008) as usize; |
| 93 | let num_hidden_layers = |
| 94 | get_u32(&content, &format!("{arch}.block_count")).unwrap_or(32) as usize; |
| 95 | let num_attention_heads = |
| 96 | get_u32(&content, &format!("{arch}.attention.head_count")).unwrap_or(32) as usize; |
| 97 | let num_key_value_heads = |
| 98 | get_u32(&content, &format!("{arch}.attention.head_count_kv")).unwrap_or(num_attention_heads as u32) as usize; |
| 99 | let rms_norm_eps = get_f32( |
| 100 | &content, |
| 101 | &format!("{arch}.attention.layer_norm_rms_epsilon"), |
| 102 | ) |
| 103 | .unwrap_or(1e-5) as f64; |
| 104 | let rope_theta = |
| 105 | get_f32(&content, &format!("{arch}.rope.freq_base")).unwrap_or(10000.0); |
| 106 | let max_seq_len = |
| 107 | get_u32(&content, &format!("{arch}.context_length")).unwrap_or(4096) as usize; |
| 108 | let vocab_size = |
| 109 | get_u32(&content, &format!("{arch}.vocab_size")) |
| 110 | .or_else(|| { |
| 111 | // Fallback: count from tokenizer tokens |
| 112 | content.tensor_infos.get("token_embd.weight") |
| 113 | .map(|ti| ti.shape.dims()[0] as u32) |
| 114 | }) |
| 115 | .unwrap_or(32000) as usize; |
| 116 | |
| 117 | // Detect QKV bias from tensor presence |
| 118 | let has_qkv_bias = content.tensor_infos.contains_key("blk.0.attn_q.bias"); |
| 119 | |
| 120 | Ok(crate::models::common::Config { |
| 121 | hidden_size, |
| 122 | intermediate_size, |
| 123 | vocab_size, |
| 124 | num_hidden_layers, |
| 125 | num_attention_heads, |
| 126 | num_key_value_heads, |
| 127 | rms_norm_eps, |
| 128 | rope_theta, |
| 129 | bos_token_id: None, |
| 130 | eos_token_id: None, |
| 131 | rope_scaling: None, |
| 132 | tie_word_embeddings: !content.tensor_infos.contains_key("output.weight"), |
| 133 | max_seq_len, |
| 134 | use_qkv_bias: has_qkv_bias, |
| 135 | model_prefix: "model".into(), |