| 214 | |
| 215 | @staticmethod |
| 216 | def loadHFTransformerJson(model: LazyModel, config_path: Path) -> Params: |
| 217 | config = json.load(open(config_path)) |
| 218 | |
| 219 | rope_scaling_type = f_rope_scale = n_orig_ctx = rope_finetuned = None |
| 220 | rope_scaling = config.get("rope_scaling") |
| 221 | |
| 222 | if rope_scaling is not None and (typ := rope_scaling.get("type")): |
| 223 | rope_factor = rope_scaling.get("factor") |
| 224 | f_rope_scale = rope_factor |
| 225 | if typ == "linear": |
| 226 | rope_scaling_type = gguf.RopeScalingType.LINEAR |
| 227 | elif typ == "yarn": |
| 228 | rope_scaling_type = gguf.RopeScalingType.YARN |
| 229 | n_orig_ctx = rope_scaling['original_max_position_embeddings'] |
| 230 | rope_finetuned = rope_scaling['finetuned'] |
| 231 | else: |
| 232 | raise NotImplementedError(f'Unknown rope scaling type: {typ}') |
| 233 | |
| 234 | if "max_sequence_length" in config: |
| 235 | n_ctx = config["max_sequence_length"] |
| 236 | elif "max_position_embeddings" in config: |
| 237 | n_ctx = config["max_position_embeddings"] |
| 238 | else: |
| 239 | raise Exception("failed to guess 'n_ctx'. This model is unknown or unsupported.\n" |
| 240 | "Suggestion: provide 'config.json' of the model in the same directory containing model files.") |
| 241 | |
| 242 | n_experts = None |
| 243 | n_experts_used = None |
| 244 | |
| 245 | if "num_local_experts" in config: |
| 246 | n_experts = config["num_local_experts"] |
| 247 | n_experts_used = config["num_experts_per_tok"] |
| 248 | |
| 249 | return Params( |
| 250 | n_vocab = config["vocab_size"], |
| 251 | n_embd = config["hidden_size"], |
| 252 | n_layer = config["num_hidden_layers"], |
| 253 | n_ctx = n_ctx, |
| 254 | n_ff = config["intermediate_size"], |
| 255 | n_head = (n_head := config["num_attention_heads"]), |
| 256 | n_head_kv = config.get("num_key_value_heads", n_head), |
| 257 | n_experts = n_experts, |
| 258 | n_experts_used = n_experts_used, |
| 259 | f_norm_eps = config["rms_norm_eps"], |
| 260 | f_rope_freq_base = config.get("rope_theta"), |
| 261 | rope_scaling_type = rope_scaling_type, |
| 262 | f_rope_scale = f_rope_scale, |
| 263 | n_orig_ctx = n_orig_ctx, |
| 264 | rope_finetuned = rope_finetuned, |
| 265 | ) |
| 266 | |
| 267 | # LLaMA v2 70B params.json |
| 268 | # {"dim": 8192, "multiple_of": 4096, "ffn_dim_multiplier": 1.3, "n_heads": 64, "n_kv_heads": 8, "n_layers": 80, "norm_eps": 1e-05, "vocab_size": -1} |